Upload 29 files

.gitattributes

@@ -25,7 +25,6 @@
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright Zhengxiao Du
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

MODEL_LICENSE

@@ -0,0 +1,33 @@
+The GLM-130B License
+
+1. Definitions
+
+“Licensor” means the GLM-130B Model Team that distributes its Software.
+
+“Software” means the GLM-130B model parameters made available under this license.
+
+2. License Grant
+
+Subject to the terms and conditions of this License, the Licensor hereby grants to you a non-exclusive, worldwide, non-transferable, non-sublicensable, revocable, royalty-free copyright license to use the Software solely for your non-commercial research purposes.
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+3. Restriction
+
+You will not use, copy, modify, merge, publish, distribute, reproduce, or create derivative works of the Software, in whole or in part, for any commercial, military, or illegal purposes.
+
+You will not use the Software for any act that may undermine China's national security and national unity, harm the public interest of society, or infringe upon the rights and interests of human beings.
+
+4. Disclaimer
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+5. Limitation of Liability
+
+EXCEPT TO THE EXTENT PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER BASED IN TORT, NEGLIGENCE, CONTRACT, LIABILITY, OR OTHERWISE WILL ANY LICENSOR BE LIABLE TO YOU FOR ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES, OR ANY OTHER COMMERCIAL LOSSES, EVEN IF THE LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+6. Dispute Resolution
+
+This license shall be governed and construed in accordance with the laws of People’s Republic of China. Any dispute arising from or in connection with this License shall be submitted to Haidian District People's Court in Beijing.
+
+Note that the license is subject to update to a more comprehensive version.  For any questions related to the license and copyright, please contact us at glm-130b@googlegroups.com.

README.md

@@ -1,145 +1,89 @@
 ---
-thumbnail: "url to a thumbnail used in social sharing"
+language:
+- zh
+- en
 tags:
-- lora
-- LLM
+- glm
 - chatglm
-- chatglm-6b
-- chatglm2-6b
-- pytorch
-- peft
-- ft
-- sft
-- PPO
-- RLHF
-- RM
-- Transformers
-license: "apache-2.0"
+- thudm
 ---
---------------------------------------------------------------------------------
-# 重磅消息
-- 本项目经过多位网友实测，中文总结能力超越了GPT3.5各版本
-# 重大突破
-- 经优化目前可以支持无限context，远大于4k、8K、16K......
-# ChatGLM-6B RLHF & LoRA Model
-
-ChatGLM-6B 是开源中英双语对话模型，本次训练基于ChatGLM-6B 的第一代版本，在保留了初代模型对话流畅、部署门槛较低等众多优秀特性的基础之上开展训练。
-## 本次训练使用的方法
-
-- 首先，用40万条高质量数据进行强化训练，以提高模型的基础能力；
-- 第二，使用30万条人类反馈数据，构建一个表达方式规范优雅的语言模式（RM模型）；
-- 第三，在保留SFT阶段三分之一训练数据的同时，增加了30万条fitness数据，叠加RM模型，对ChatGLM-6B进行强化训练。
-- 成果，训练后在健康咨询，文档总结能力上不但强于chatglm-6b，而且部分能力上更是强于chatglm2-6b，配合“闻达”和“langchain-chatglm”等知识库项目，应用体验上对比chatglm-6b、chatglm2-6b和百川-7b均匀显著提升。
-- 性能，fp16运行时速度上比原模型提升20%.可以代替原有官方模型，大家可以fp16、int4、int8使用。
-- 兼容性，本项目全模型的运行方式与原模型一致。lora文件运行方式，建议在原模型chatglm-6b上运行，在chatglm2-6b上可以正常加载但不推荐，只有当上下文大于4k时在chatglm2-6b上运行有一定价值，经（网友：大笨熊）测试有一定效果，但是效果不能完全发挥。
-- 特性，基于模型对自然对话的超强理解力和总结能力，连续会话不受tokens限制，支持无限轮次的智能对话。
-- 协议
-- 本仓库的代码依照 Apache-2.0 协议开源，ChatGLM2-6B 模型的权重的使用则需要遵循 Model License。
-- 授权方式，与原项目一致，未经过chatglm-6b原开发方允许，不得用于商业用途。详细见原项目相关规定，模型地址https://huggingface.co/THUDM/chatglm-6b
-- 本次训练由智能AI用户[帛凡]于2023年基于ChatGLM-6b进行独立完成。（严禁售卖或者商业项目，任何通过此项目产生的知识仅用于参考，作者不承担任何责任）。
-- 百度网盘 https://pan.baidu.com/s/1l9q_7h8nGdelIwYlCbllMg?pwd=klhu （感谢网友 ：宋小猫 提供分享）
-- 夸克网盘 https://pan.quark.cn/s/d947c6dbf592
-
-- 原模型量化评测
-![原模型量化评测](glm_eval.jpg)
-- 训练后量化评测
-![训练后量化评测](lora_eva.jpg)
-## Usage1 16G及以上显存用下载压缩包即lora文件使用，可支持ChatGLM原生模型和LoRA微调后的模型
-16G及以上显存用下载压缩包即lora文件使用，可支持ChatGLM原生模型和LoRA微调后的模型
-(HuggingFace Transformers)
-First, you pass your input through the transformer model, then you get the generated sentence.
-Install package:
-```
-pip install transformers 
-```
+# ChatGLM-6B
+<p align="center">
+   🌐 <a href="https://chatglm.cn/blog" target="_blank">Blog</a> • 💻 <a href="https://github.com/THUDM/ChatGLM-6B" target="_blank">Github Repo</a> • 🐦 <a href="https://twitter.com/thukeg" target="_blank">Twitter</a> • 📃 <a href="https://arxiv.org/abs/2103.10360" target="_blank">[GLM@ACL 22]</a> <a href="https://github.com/THUDM/GLM" target="_blank">[GitHub]</a> • 📃 <a href="https://arxiv.org/abs/2210.02414" target="_blank">[GLM-130B@ICLR 23]</a> <a href="https://github.com/THUDM/GLM-130B" target="_blank">[GitHub]</a> <br>
+</p>
+
+<p align="center">
+    👋 Join our <a href="https://join.slack.com/t/chatglm/shared_invite/zt-1udqapmrr-ocT1DS_mxWe6dDY8ahRWzg" target="_blank">Slack</a> and <a href="https://github.com/THUDM/ChatGLM-6B/blob/main/resources/WECHAT.md" target="_blank">WeChat</a>
+</p>
+
+## 介绍
+ChatGLM-6B 是一个开源的、支持中英双语问答的对话语言模型，基于 [General Language Model (GLM)](https://github.com/THUDM/GLM) 架构，具有 62 亿参数。结合模型量化技术，用户可以在消费级的显卡上进行本地部署（INT4 量化级别下最低只需 6GB 显存）。ChatGLM-6B 使用了和 [ChatGLM](https://chatglm.cn) 相同的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答。
+
+ChatGLM-6B is an open bilingual language model based on [General Language Model (GLM)](https://github.com/THUDM/GLM) framework, with 6.2 billion parameters. With the quantization technique, users can deploy locally on consumer-grade graphics cards (only 6GB of GPU memory is required at the INT4 quantization level). ChatGLM-6B uses technology similar to ChatGPT, optimized for Chinese QA and dialogue. The model is trained for about 1T tokens of Chinese and English corpus, supplemented by supervised fine-tuning, feedback bootstrap, and reinforcement learning wit human feedback. With only about 6.2 billion parameters, the model is able to generate answers that are in line with human preference.
+
+## 软件依赖
 
-```python
-
-import sys
-from peft import PeftModel
-from transformers import AutoModel, AutoTokenizer
-sys.path.append('..')
-model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True, device_map='auto')
-model = PeftModel.from_pretrained(model, "model/chatglm_fitness_lora")
-model = model.half().cuda()  # fp16
-tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
-sents = ['新冠肺炎怎么预防。\n答：']
-for s in sents:
-    response = model.chat(tokenizer, s, max_length=128, eos_token_id=tokenizer.eos_token_id)
-    print(response)
-```
-output:
 ```shell
-例一：常识综合问答
-问： 新冠肺炎怎么预防
-答：新冠肺炎是一种由新型冠状病毒引起的疾病，可以通过飞沫传播和接触传播。以下是一些预防新冠肺炎的建议：
-戴口罩：佩戴口罩可以防止呼吸道飞沫传播，减少感染风险。在公共场合、公共交通工具上、与他人交往时，应佩戴医用口罩或N95口罩。
-勤洗手：经常用肥皂和温水洗手，尤其在接触公共场所、物品或与他人交往后，或咳嗽、打喷嚏、擤鼻涕后等。
-保持社交距离：尽量避免近距离接触他人，保持至少1米的社交距离。
-避免聚集：尽量避免参加人员密集的聚会、活动或旅行。
-加强室内通风：在室内保持良好的通风可以降低病毒在空气中的浓度。
-健康生活方式：保持充足的睡眠、均衡的饮食、适量的运动等健康生活方式可以增强身体免疫力，预防感染。
-接种疫苗：疫苗接种是预防新冠肺炎最有效的方法之一。如果有机会，应接种疫苗。
-预防新冠肺炎需要个人和社会的共同努力，遵守公共卫生规定，加强个人防护，减少接触风险。
-例二：较为专业的医疗和心理学问答
-问：系统性红斑狼疮的危害和治疗方法是什么？
-答：系统性红斑狼疮(Systemic Lupus Erythematosus,SLE)是一种自身免疫性疾病，可以影响身体的多个器官和组织。SLE的危害包括：
-1. 心血管疾病：SLE可以引起高血压、心脏病和中风等心血管疾病。
-2. 肾脏疾病：SLE可以引起肾小球肾炎和肾衰竭等肾脏疾病。
-3. 口腔溃疡：SLE可以引起口腔和舌头溃疡等口腔疾病。
-4. 皮肤疾病：SLE可以引起皮疹、红斑狼疮面容和光敏感等皮肤疾病。
-
-治疗方法包括：
-1. 药物治疗：SLE可以使用非甾体抗炎药、激素类药物和免疫抑制剂等药物治疗。
-2. 支持性治疗：SLE可以使用营养支持、心理治疗和支持性护理等支持性治疗。
-3. 手术治疗：在一些情况下，SLE可能需要手术治疗，如肾脏移植等。
-
-SLE是一种严重的疾病，需要患者及时接受专业治疗。患者需要积极配合医生的治疗，保持良好的生活习惯和饮食习惯，定期进行检查和监测。
-例三：无限多轮次的智能对话（第一轮见例二）
-问：早期激素和免疫抑制剂用药建议是什么？
-答：系统性红斑狼疮的早期治疗非常重要，因为病情的严重程度和进展速度可能会影响患者的生活质量和生命安全。对于系统性红斑狼疮的早期治疗，激素和免疫抑制剂的使用是非常重要的。以下是一些早期激素和免疫抑制剂用药的建议：
-1. 激素类药物：激素类药物是系统性红斑狼疮的主要治疗方法之一。在使用激素类药物时，需要遵循医生的建议，根据病情和身体状况进行剂量和用药时间的调整。一般来说，激素类药物需要长期使用，以避免药物副作用和病情的复发。
-2. 免疫抑制剂：免疫抑制剂是系统性红斑狼疮另一种重要的治疗方法。在使用免疫抑制剂时，需要遵循医生的建议，根据病情和身体状况进行用药时间和剂量的调整。免疫抑制剂可以用于治疗系统性红斑狼疮的各种症状和并发症，如肾脏疾病、口腔溃疡、皮肤疾病等。
-3. 其他治疗：除了激素和免疫抑制剂之外，系统性红斑狼疮的治疗还包括其他治疗方法，如支持性治疗、手术治疗等。在使用这些治疗方法时，需要遵循医生的建议，根据病情和身体状况进行用药和治疗。
-
-早期激素和免疫抑制剂的使用非常重要，需要患者积极配合医生的治疗，遵循医生的建议，定期进行检查和监测，以确保病情得到有效控制和生活质量得到保障。
+pip install protobuf==3.20.0 transformers==4.27.1 icetk cpm_kernels
 ```
 
-模型文件组成：
-```
-chatglm_fitness_lora
-    ├── adapter_config.json
-    └── adapter_model.bin
-```
---------------------------------------------------------------------------------
-## Usage2 16G及以下显存用户下载整个模型，可支持fp16、int8、int4
-First, you pass your input through the transformer model, then you get the generated sentence.
-```
-pip install transformers 
+## 代码调用 
+
+可以通过如下代码调用 ChatGLM-6B 模型来生成对话：
+
+```ipython
+>>> from transformers import AutoTokenizer, AutoModel
+>>> tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
+>>> model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda()
+>>> response, history = model.chat(tokenizer, "你好", history=[])
+>>> print(response)
+你好👋!我是人工智能助手 ChatGLM-6B,很高兴见到你,欢迎问我任何问题。
+>>> response, history = model.chat(tokenizer, "晚上睡不着应该怎么办", history=history)
+>>> print(response)
+晚上睡不着可能会让你感到焦虑或不舒服,但以下是一些可以帮助你入睡的方法:
+
+1. 制定规律的睡眠时间表:保持规律的睡眠时间表可以帮助你建立健康的睡眠习惯,使你更容易入睡。尽量在每天的相同时间上床,并在同一时间起床。
+2. 创造一个舒适的睡眠环境:确保睡眠环境舒适,安静,黑暗且温度适宜。可以使用舒适的床上用品,并保持房间通风。
+3. 放松身心:在睡前做些放松的活动,例如泡个热水澡,听些轻柔的音乐,阅读一些有趣的书籍等,有助于缓解紧张和焦虑,使你更容易入睡。
+4. 避免饮用含有咖啡因的饮料:咖啡因是一种刺激性物质,会影响你的睡眠质量。尽量避免在睡前饮用含有咖啡因的饮料,例如咖啡,茶和可乐。
+5. 避免在床上做与睡眠无关的事情:在床上做些与睡眠无关的事情,例如看电影,玩游戏或工作等,可能会干扰你的睡眠。
+6. 尝试呼吸技巧:深呼吸是一种放松技巧,可以帮助你缓解紧张和焦虑,使你更容易入睡。试着慢慢吸气,保持几秒钟,然后缓慢呼气。
+
+如果这些方法无法帮助你入睡,你可以考虑咨询医生或睡眠专家,寻求进一步的建议。
 ```
 
-```python
-
-import sys
-from peft import PeftModel
-from transformers import AutoModel, AutoTokenizer
-sys.path.append('..')
-model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True, device_map='auto')
-model = PeftModel.from_pretrained(model, "model/chatglm_fitness_lora") # lora文件保存目录
-model = model.half().quantize(4).cuda()  # int4
-#model = model.half().quantize(8).cuda()  # int8
-#model = model.half().cuda()  # fp16
-tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
-sents = ['新冠肺炎怎么预防。\n答：']
-for s in sents:
-    response = model.chat(tokenizer, s, max_length=128, eos_token_id=tokenizer.eos_token_id)
-    print(response)
+关于更多的使用说明，包括如何运行命令行和网页版本的 DEMO，以及使用模型量化以节省显存，请参考我们的 [Github Repo](https://github.com/THUDM/ChatGLM-6B)。
+
+For more instructions, including how to run CLI and web demos, and model quantization, please refer to our [Github Repo](https://github.com/THUDM/ChatGLM-6B).
+
+## Change Log
+* v1.1.0 ([942945d](https://huggingface.co/THUDM/chatglm-6b/commit/942945df047dee66f653c68ae0e56655045f1741)): 更新 v1.1 版本 checkpoint
+* v0.1.0 ([f831824](https://huggingface.co/THUDM/chatglm-6b/commit/f83182484538e663a03d3f73647f10f89878f438))
+
+## 协议
+
+本仓库的代码依照 [Apache-2.0](LICENSE) 协议开源，ChatGLM-6B 模型的权重的使用则需要遵循 [Model License](MODEL_LICENSE)。
+
+## 引用
+
+如果你觉得我们的工作有帮助的话，请考虑引用下列论文：
+
+```
+@inproceedings{
+  zeng2023glm-130b,
+  title={{GLM}-130B: An Open Bilingual Pre-trained Model},
+  author={Aohan Zeng and Xiao Liu and Zhengxiao Du and Zihan Wang and Hanyu Lai and Ming Ding and Zhuoyi Yang and Yifan Xu and Wendi Zheng and Xiao Xia and Weng Lam Tam and Zixuan Ma and Yufei Xue and Jidong Zhai and Wenguang Chen and Zhiyuan Liu and Peng Zhang and Yuxiao Dong and Jie Tang},
+  booktitle={The Eleventh International Conference on Learning Representations (ICLR)},
+  year={2023},
+  url={https://openreview.net/forum?id=-Aw0rrrPUF}
+}
 ```
-output:
-```shell
-例四：优于chatglm-6b、chatglm2-6b和百川-7b等类似参数量模型的总结归纳能力
-问：请用简短的语言总结下面的文字：
-大语言模型是指能够生成、理解和处理自然语言的高度智能化的计算机模型。这些模型使用深度学习技术，尤其是循环神经网络（RNN）或变种，如长短期记忆（LSTM）或注意力机制（attention mechanism），从大规模文本语料库中进行训练。
-大语言模型的训练过程通常基于预测下一个单词或字符的任务。通过对大量文本数据进行训练，模型能够学习到语言的潜在��式、结构和语义含义。这使得大语言模型能够产生流畅、连贯的文本，回答问题，完成翻译任务，生成代码等。
-答：大语言模型是一种使用深度学习技术训练的计算机模型，能够生成、理解和处理自然语言。通过训练大量文本数据，大语言模型能够产生流畅、连贯的文本，回答问题，完成翻译任务，生成代码等。
 ```
+@inproceedings{du2022glm,
+  title={GLM: General Language Model Pretraining with Autoregressive Blank Infilling},
+  author={Du, Zhengxiao and Qian, Yujie and Liu, Xiao and Ding, Ming and Qiu, Jiezhong and Yang, Zhilin and Tang, Jie},
+  booktitle={Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
+  pages={320--335},
+  year={2022}
+}
+```

config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "D:\\glm\\chatglm_webui\\chatglm-6b",
+  "architectures": [
+    "ChatGLMForConditionalGeneration"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_chatglm.ChatGLMConfig",
+    "AutoModel": "modeling_chatglm.ChatGLMForConditionalGeneration",
+    "AutoModelForSeq2SeqLM": "modeling_chatglm.ChatGLMForConditionalGeneration"
+  },
+  "bos_token_id": 130004,
+  "eos_token_id": 130005,
+  "gmask_token_id": 130001,
+  "hidden_size": 4096,
+  "inner_hidden_size": 16384,
+  "layernorm_epsilon": 1e-05,
+  "mask_token_id": 130000,
+  "max_sequence_length": 2048,
+  "model_type": "chatglm",
+  "num_attention_heads": 32,
+  "num_layers": 28,
+  "pad_token_id": 3,
+  "position_encoding_2d": true,
+  "pre_seq_len": null,
+  "prefix_projection": false,
+  "quantization_bit": 0,
+  "torch_dtype": "float16",
+  "transformers_version": "4.30.0",
+  "use_cache": true,
+  "vocab_size": 130528
+}

configuration_chatglm.py

@@ -0,0 +1,103 @@
+""" ChatGLM model configuration """
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+class ChatGLMConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`~ChatGLMModel`].
+    It is used to instantiate an ChatGLM model according to the specified arguments, defining the model
+    architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
+    the ChatGLM-6B [THUDM/ChatGLM-6B](https://huggingface.co/THUDM/chatglm-6b) architecture.
+
+    Configuration objects inherit from  [`PretrainedConfig`] and can be used
+    to control the model outputs. Read the documentation from  [`PretrainedConfig`]
+    for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 150528):
+            Vocabulary size of the ChatGLM-6B model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`~ChatGLMModel`] or
+            [`~TFChatGLMModel`].
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the encoder layers and the pooler layer.
+        num_hidden_layers (`int`, *optional*, defaults to 28):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        inner_hidden_size (`int`, *optional*, defaults to 16384):
+            Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+        max_sequence_length (`int`, *optional*, defaults to 512):
+            The maximum sequence length that this model might ever be used with.
+            Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+        layernorm_epsilon (`float`, *optional*, defaults to 1e-5):
+            The epsilon used by the layer normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether the model should return the last key/values attentions (not used by all models).
+        Example:
+
+    ```python
+    >>> from configuration_chatglm import ChatGLMConfig
+    >>> from modeling_chatglm import ChatGLMModel
+
+    >>> # Initializing a ChatGLM-6B THUDM/ChatGLM-6B style configuration
+    >>> configuration = ChatGLMConfig()
+
+    >>> # Initializing a model from the THUDM/ChatGLM-6B style configuration
+    >>> model = ChatGLMModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```
+"""
+    model_type = "chatglm"
+
+    def __init__(
+            self,
+            vocab_size=150528,
+            hidden_size=4096,
+            num_layers=28,
+            num_attention_heads=32,
+            layernorm_epsilon=1e-5,
+            use_cache=False,
+            bos_token_id=150004,
+            eos_token_id=150005,
+            mask_token_id=150000,
+            gmask_token_id=150001,
+            pad_token_id=0,
+            max_sequence_length=2048,
+            inner_hidden_size=16384,
+            position_encoding_2d=True,
+            quantization_bit=0,
+            pre_seq_len=None,
+            prefix_projection=False,
+            **kwargs
+    ):
+        self.num_layers = num_layers
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_attention_heads = num_attention_heads
+        self.max_sequence_length = max_sequence_length
+        self.layernorm_epsilon = layernorm_epsilon
+        self.inner_hidden_size = inner_hidden_size
+        self.use_cache = use_cache
+        self.bos_token_id = bos_token_id
+        self.eos_token_id = eos_token_id
+        self.pad_token_id = pad_token_id
+        self.mask_token_id = mask_token_id
+        self.gmask_token_id = gmask_token_id
+        self.position_encoding_2d = position_encoding_2d
+        self.quantization_bit = quantization_bit
+        self.pre_seq_len = pre_seq_len
+        self.prefix_projection = prefix_projection
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            **kwargs
+        )

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 130004,
+  "eos_token_id": 130005,
+  "pad_token_id": 3,
+  "transformers_version": "4.30.0"
+}

ice_text.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e974d9a69c242ce014c88c2b26089270f6198f3c0b700a887666cd3e816f17e
+size 2706249

modeling_chatglm.py

@@ -0,0 +1,1435 @@
+""" PyTorch ChatGLM model. """
+
+import math
+import copy
+import os
+import warnings
+import re
+import sys
+
+import torch
+import torch.utils.checkpoint
+import torch.nn.functional as F
+from torch import nn
+from torch.nn import CrossEntropyLoss, LayerNorm
+from torch.nn.utils import skip_init
+from typing import Optional, Tuple, Union, List, Callable, Dict, Any
+
+from transformers.utils import (
+    add_code_sample_docstrings,
+    add_start_docstrings,
+    add_start_docstrings_to_model_forward,
+)
+from transformers.modeling_outputs import (
+    BaseModelOutputWithPast,
+    CausalLMOutputWithPast,
+    BaseModelOutputWithPastAndCrossAttentions,
+)
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import logging
+from transformers.generation.logits_process import LogitsProcessor
+from transformers.generation.utils import LogitsProcessorList, StoppingCriteriaList, GenerationConfig, ModelOutput
+
+from .configuration_chatglm import ChatGLMConfig
+
+# flags required to enable jit fusion kernels
+
+if sys.platform != 'darwin':
+    torch._C._jit_set_profiling_mode(False)
+    torch._C._jit_set_profiling_executor(False)
+    torch._C._jit_override_can_fuse_on_cpu(True)
+    torch._C._jit_override_can_fuse_on_gpu(True)
+
+logger = logging.get_logger(__name__)
+
+_CHECKPOINT_FOR_DOC = "THUDM/ChatGLM-6B"
+_CONFIG_FOR_DOC = "ChatGLM6BConfig"
+
+CHATGLM_6B_PRETRAINED_MODEL_ARCHIVE_LIST = [
+    "THUDM/chatglm-6b",
+    # See all ChatGLM-6B models at https://huggingface.co/models?filter=chatglm
+]
+
+
+class InvalidScoreLogitsProcessor(LogitsProcessor):
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
+        if torch.isnan(scores).any() or torch.isinf(scores).any():
+            scores.zero_()
+            scores[..., 5] = 5e4
+        return scores
+
+
+def load_tf_weights_in_chatglm_6b(model, config, tf_checkpoint_path):
+    """Load tf checkpoints in a pytorch model."""
+    try:
+        import re
+
+        import numpy as np
+        import tensorflow as tf
+    except ImportError:
+        logger.error(
+            "Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see "
+            "https://www.tensorflow.org/install/ for installation instructions."
+        )
+        raise
+    tf_path = os.path.abspath(tf_checkpoint_path)
+    logger.info(f"Converting TensorFlow checkpoint from {tf_path}")
+    # Load weights from TF model
+    init_vars = tf.train.list_variables(tf_path)
+    names = []
+    arrays = []
+    for name, shape in init_vars:
+        logger.info(f"Loading TF weight {name} with shape {shape}")
+        array = tf.train.load_variable(tf_path, name)
+        names.append(name)
+        arrays.append(array)
+
+    for name, array in zip(names, arrays):
+        name = name.split("/")
+        # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v
+        # which are not required for using pretrained model
+        if any(
+                n in ["adam_v", "adam_m", "AdamWeightDecayOptimizer", "AdamWeightDecayOptimizer_1", "global_step"]
+                for n in name
+        ):
+            logger.info(f"Skipping {'/'.join(name)}")
+            continue
+        pointer = model
+        for m_name in name:
+            if re.fullmatch(r"[A-Za-z]+_\d+", m_name):
+                scope_names = re.split(r"_(\d+)", m_name)
+            else:
+                scope_names = [m_name]
+            if scope_names[0] == "kernel" or scope_names[0] == "gamma":
+                pointer = getattr(pointer, "weight")
+            elif scope_names[0] == "output_bias" or scope_names[0] == "beta":
+                pointer = getattr(pointer, "bias")
+            elif scope_names[0] == "output_weights":
+                pointer = getattr(pointer, "weight")
+            elif scope_names[0] == "squad":
+                pointer = getattr(pointer, "classifier")
+            else:
+                try:
+                    pointer = getattr(pointer, scope_names[0])
+                except AttributeError:
+                    logger.info(f"Skipping {'/'.join(name)}")
+                    continue
+            if len(scope_names) >= 2:
+                num = int(scope_names[1])
+                pointer = pointer[num]
+        if m_name[-11:] == "_embeddings":
+            pointer = getattr(pointer, "weight")
+        elif m_name == "kernel":
+            array = np.transpose(array)
+        try:
+            assert (
+                    pointer.shape == array.shape
+            ), f"Pointer shape {pointer.shape} and array shape {array.shape} mismatched"
+        except AssertionError as e:
+            e.args += (pointer.shape, array.shape)
+            raise
+        logger.info(f"Initialize PyTorch weight {name}")
+        pointer.data = torch.from_numpy(array)
+    return model
+
+
+class PrefixEncoder(torch.nn.Module):
+    """
+    The torch.nn model to encode the prefix
+    Input shape: (batch-size, prefix-length)
+    Output shape: (batch-size, prefix-length, 2*layers*hidden)
+    """
+
+    def __init__(self, config):
+        super().__init__()
+        self.prefix_projection = config.prefix_projection
+        if self.prefix_projection:
+            # Use a two-layer MLP to encode the prefix
+            self.embedding = torch.nn.Embedding(config.pre_seq_len, config.hidden_size)
+            self.trans = torch.nn.Sequential(
+                torch.nn.Linear(config.hidden_size, config.hidden_size),
+                torch.nn.Tanh(),
+                torch.nn.Linear(config.hidden_size, config.num_layers * config.hidden_size * 2)
+            )
+        else:
+            self.embedding = torch.nn.Embedding(config.pre_seq_len, config.num_layers * config.hidden_size * 2)
+
+    def forward(self, prefix: torch.Tensor):
+        if self.prefix_projection:
+            prefix_tokens = self.embedding(prefix)
+            past_key_values = self.trans(prefix_tokens)
+        else:
+            past_key_values = self.embedding(prefix)
+        return past_key_values
+
+
+@torch.jit.script
+def gelu_impl(x):
+    """OpenAI's gelu implementation."""
+    return 0.5 * x * (1.0 + torch.tanh(0.7978845608028654 * x *
+                                       (1.0 + 0.044715 * x * x)))
+
+
+def gelu(x):
+    return gelu_impl(x)
+
+
+class RotaryEmbedding(torch.nn.Module):
+    def __init__(self, dim, base=10000, precision=torch.half, learnable=False):
+        super().__init__()
+        inv_freq = 1. / (base ** (torch.arange(0, dim, 2).float() / dim))
+        inv_freq = inv_freq.half()
+        self.learnable = learnable
+        if learnable:
+            self.inv_freq = torch.nn.Parameter(inv_freq)
+            self.max_seq_len_cached = None
+        else:
+            self.register_buffer('inv_freq', inv_freq)
+            self.max_seq_len_cached = None
+            self.cos_cached = None
+            self.sin_cached = None
+        self.precision = precision
+
+    def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys,
+                              error_msgs):
+        pass
+
+    def forward(self, x, seq_dim=1, seq_len=None):
+        if seq_len is None:
+            seq_len = x.shape[seq_dim]
+        if self.max_seq_len_cached is None or (seq_len > self.max_seq_len_cached):
+            self.max_seq_len_cached = None if self.learnable else seq_len
+            t = torch.arange(seq_len, device=x.device, dtype=self.inv_freq.dtype)
+            freqs = torch.einsum('i,j->ij', t, self.inv_freq)
+            # Different from paper, but it uses a different permutation in order to obtain the same calculation
+            emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
+            if self.precision == torch.bfloat16:
+                emb = emb.float()
+
+            # [sx, 1 (b * np), hn]
+            cos_cached = emb.cos()[:, None, :]
+            sin_cached = emb.sin()[:, None, :]
+            if self.precision == torch.bfloat16:
+                cos_cached = cos_cached.bfloat16()
+                sin_cached = sin_cached.bfloat16()
+            if self.learnable:
+                return cos_cached, sin_cached
+            self.cos_cached, self.sin_cached = cos_cached, sin_cached
+        return self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...]
+
+    def _apply(self, fn):
+        if self.cos_cached is not None:
+            self.cos_cached = fn(self.cos_cached)
+        if self.sin_cached is not None:
+            self.sin_cached = fn(self.sin_cached)
+        return super()._apply(fn)
+
+
+def rotate_half(x):
+    x1, x2 = x[..., :x.shape[-1] // 2], x[..., x.shape[-1] // 2:]
+    return torch.cat((-x2, x1), dim=x1.ndim - 1)  # dim=-1 triggers a bug in earlier torch versions
+
+
+@torch.jit.script
+def apply_rotary_pos_emb_index(q, k, cos, sin, position_id):
+    # position_id: [sq, b], q, k: [sq, b, np, hn], cos: [sq, 1, hn] -> [sq, b, 1, hn]
+    cos, sin = F.embedding(position_id, cos.squeeze(1)).unsqueeze(2), \
+        F.embedding(position_id, sin.squeeze(1)).unsqueeze(2)
+    q, k = (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin)
+    return q, k
+
+
+def attention_fn(
+        self,
+        query_layer,
+        key_layer,
+        value_layer,
+        attention_mask,
+        hidden_size_per_partition,
+        layer_id,
+        layer_past=None,
+        scaling_attention_score=True,
+        use_cache=False,
+):
+    if layer_past is not None:
+        past_key, past_value = layer_past[0], layer_past[1]
+        key_layer = torch.cat((past_key, key_layer), dim=0)
+        value_layer = torch.cat((past_value, value_layer), dim=0)
+
+    # seqlen, batch, num_attention_heads, hidden_size_per_attention_head
+    seq_len, b, nh, hidden_size = key_layer.shape
+
+    if use_cache:
+        present = (key_layer, value_layer)
+    else:
+        present = None
+
+    query_key_layer_scaling_coeff = float(layer_id + 1)
+    if scaling_attention_score:
+        query_layer = query_layer / (math.sqrt(hidden_size) * query_key_layer_scaling_coeff)
+
+    # ===================================
+    # Raw attention scores. [b, np, s, s]
+    # ===================================
+
+    # [b, np, sq, sk]
+    output_size = (query_layer.size(1), query_layer.size(2), query_layer.size(0), key_layer.size(0))
+
+    # [sq, b, np, hn] -> [sq, b * np, hn]
+    query_layer = query_layer.view(output_size[2], output_size[0] * output_size[1], -1)
+    # [sk, b, np, hn] -> [sk, b * np, hn]
+    key_layer = key_layer.view(output_size[3], output_size[0] * output_size[1], -1)
+
+    matmul_result = torch.zeros(
+        1, 1, 1,
+        dtype=query_layer.dtype,
+        device=query_layer.device,
+    )
+
+    matmul_result = torch.baddbmm(
+        matmul_result,
+        query_layer.transpose(0, 1),  # [b * np, sq, hn]
+        key_layer.transpose(0, 1).transpose(1, 2),  # [b * np, hn, sk]
+        beta=0.0,
+        alpha=1.0,
+    )
+
+    # change view to [b, np, sq, sk]
+    attention_scores = matmul_result.view(*output_size)
+
+    if self.scale_mask_softmax:
+        self.scale_mask_softmax.scale = query_key_layer_scaling_coeff
+        attention_probs = self.scale_mask_softmax(attention_scores, attention_mask.contiguous())
+    else:
+        if not (attention_mask == 0).all():
+            # if auto-regressive, skip
+            attention_scores.masked_fill_(attention_mask, -10000.0)
+        dtype = attention_scores.dtype
+        attention_scores = attention_scores.float()
+        attention_scores = attention_scores * query_key_layer_scaling_coeff
+
+        attention_probs = F.softmax(attention_scores, dim=-1)
+
+        attention_probs = attention_probs.type(dtype)
+
+    # =========================
+    # Context layer. [sq, b, hp]
+    # =========================
+
+    # value_layer -> context layer.
+    # [sk, b, np, hn] --> [b, np, sq, hn]
+
+    # context layer shape: [b, np, sq, hn]
+    output_size = (value_layer.size(1), value_layer.size(2), query_layer.size(0), value_layer.size(3))
+
+    # change view [sk, b * np, hn]
+    value_layer = value_layer.view(value_layer.size(0), output_size[0] * output_size[1], -1)
+
+    # change view [b * np, sq, sk]
+    attention_probs = attention_probs.view(output_size[0] * output_size[1], output_size[2], -1)
+
+    # matmul: [b * np, sq, hn]
+    context_layer = torch.bmm(attention_probs, value_layer.transpose(0, 1))
+
+    # change view [b, np, sq, hn]
+    context_layer = context_layer.view(*output_size)
+
+    # [b, np, sq, hn] --> [sq, b, np, hn]
+    context_layer = context_layer.permute(2, 0, 1, 3).contiguous()
+
+    # [sq, b, np, hn] --> [sq, b, hp]
+    new_context_layer_shape = context_layer.size()[:-2] + (hidden_size_per_partition,)
+    context_layer = context_layer.view(*new_context_layer_shape)
+
+    outputs = (context_layer, present, attention_probs)
+
+    return outputs
+
+
+def default_init(cls, *args, **kwargs):
+    return cls(*args, **kwargs)
+
+
+class SelfAttention(torch.nn.Module):
+    def __init__(self, hidden_size, num_attention_heads,
+                 layer_id, hidden_size_per_attention_head=None, bias=True,
+                 params_dtype=torch.float, position_encoding_2d=True, empty_init=True):
+        if empty_init:
+            init_method = skip_init
+        else:
+            init_method = default_init
+        super(SelfAttention, self).__init__()
+
+        self.layer_id = layer_id
+        self.hidden_size = hidden_size
+        self.hidden_size_per_partition = hidden_size
+        self.num_attention_heads = num_attention_heads
+        self.num_attention_heads_per_partition = num_attention_heads
+        self.position_encoding_2d = position_encoding_2d
+        self.rotary_emb = RotaryEmbedding(
+            self.hidden_size // (self.num_attention_heads * 2)
+            if position_encoding_2d
+            else self.hidden_size // self.num_attention_heads,
+            base=10000,
+            precision=torch.half,
+            learnable=False,
+        )
+
+        self.scale_mask_softmax = None
+
+        if hidden_size_per_attention_head is None:
+            self.hidden_size_per_attention_head = hidden_size // num_attention_heads
+        else:
+            self.hidden_size_per_attention_head = hidden_size_per_attention_head
+
+        self.inner_hidden_size = num_attention_heads * self.hidden_size_per_attention_head
+
+        # Strided linear layer.
+        self.query_key_value = init_method(
+            torch.nn.Linear,
+            hidden_size,
+            3 * self.inner_hidden_size,
+            bias=bias,
+            dtype=params_dtype,
+        )
+
+        self.dense = init_method(
+            torch.nn.Linear,
+            self.inner_hidden_size,
+            hidden_size,
+            bias=bias,
+            dtype=params_dtype,
+        )
+
+    @staticmethod
+    def attention_mask_func(attention_scores, attention_mask):
+        attention_scores.masked_fill_(attention_mask, -10000.0)
+        return attention_scores
+
+    def split_tensor_along_last_dim(self, tensor, num_partitions,
+                                    contiguous_split_chunks=False):
+        """Split a tensor along its last dimension.
+        Arguments:
+            tensor: input tensor.
+            num_partitions: number of partitions to split the tensor
+            contiguous_split_chunks: If True, make each chunk contiguous
+                                    in memory.
+        """
+        # Get the size and dimension.
+        last_dim = tensor.dim() - 1
+        last_dim_size = tensor.size()[last_dim] // num_partitions
+        # Split.
+        tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
+        # Note: torch.split does not create contiguous tensors by default.
+        if contiguous_split_chunks:
+            return tuple(chunk.contiguous() for chunk in tensor_list)
+
+        return tensor_list
+
+    def forward(
+            self,
+            hidden_states: torch.Tensor,
+            position_ids,
+            attention_mask: torch.Tensor,
+            layer_id,
+            layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+            use_cache: bool = False,
+            output_attentions: bool = False,
+    ):
+        """
+        hidden_states: [seq_len, batch, hidden_size]
+        attention_mask: [(1, 1), seq_len, seq_len]
+        """
+
+        # [seq_len, batch, 3 * hidden_size]
+        mixed_raw_layer = self.query_key_value(hidden_states)
+
+        # [seq_len, batch, 3 * hidden_size] --> [seq_len, batch, num_attention_heads, 3 * hidden_size_per_attention_head]
+        new_tensor_shape = mixed_raw_layer.size()[:-1] + (
+            self.num_attention_heads_per_partition,
+            3 * self.hidden_size_per_attention_head,
+        )
+        mixed_raw_layer = mixed_raw_layer.view(*new_tensor_shape)
+
+        # [seq_len, batch, num_attention_heads, hidden_size_per_attention_head]
+        (query_layer, key_layer, value_layer) = self.split_tensor_along_last_dim(mixed_raw_layer, 3)
+
+        if self.position_encoding_2d:
+            q1, q2 = query_layer.chunk(2, dim=(query_layer.ndim - 1))
+            k1, k2 = key_layer.chunk(2, dim=(key_layer.ndim - 1))
+            cos, sin = self.rotary_emb(q1, seq_len=position_ids.max() + 1)
+            position_ids, block_position_ids = position_ids[:, 0, :].transpose(0, 1).contiguous(), \
+                position_ids[:, 1, :].transpose(0, 1).contiguous()
+            q1, k1 = apply_rotary_pos_emb_index(q1, k1, cos, sin, position_ids)
+            q2, k2 = apply_rotary_pos_emb_index(q2, k2, cos, sin, block_position_ids)
+            query_layer = torch.concat([q1, q2], dim=(q1.ndim - 1))
+            key_layer = torch.concat([k1, k2], dim=(k1.ndim - 1))
+        else:
+            position_ids = position_ids.transpose(0, 1)
+            cos, sin = self.rotary_emb(value_layer, seq_len=position_ids.max() + 1)
+            # [seq_len, batch, num_attention_heads, hidden_size_per_attention_head]
+            query_layer, key_layer = apply_rotary_pos_emb_index(query_layer, key_layer, cos, sin, position_ids)
+
+        # [seq_len, batch, hidden_size]
+        context_layer, present, attention_probs = attention_fn(
+            self=self,
+            query_layer=query_layer,
+            key_layer=key_layer,
+            value_layer=value_layer,
+            attention_mask=attention_mask,
+            hidden_size_per_partition=self.hidden_size_per_partition,
+            layer_id=layer_id,
+            layer_past=layer_past,
+            use_cache=use_cache
+        )
+
+        output = self.dense(context_layer)
+
+        outputs = (output, present)
+
+        if output_attentions:
+            outputs += (attention_probs,)
+
+        return outputs  # output, present, attention_probs
+
+
+class GEGLU(torch.nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.activation_fn = F.gelu
+
+    def forward(self, x):
+        # dim=-1 breaks in jit for pt<1.10
+        x1, x2 = x.chunk(2, dim=(x.ndim - 1))
+        return x1 * self.activation_fn(x2)
+
+
+class GLU(torch.nn.Module):
+    def __init__(self, hidden_size, inner_hidden_size=None,
+                 layer_id=None, bias=True, activation_func=gelu, params_dtype=torch.float, empty_init=True):
+        super(GLU, self).__init__()
+        if empty_init:
+            init_method = skip_init
+        else:
+            init_method = default_init
+        self.layer_id = layer_id
+        self.activation_func = activation_func
+
+        # Project to 4h.
+        self.hidden_size = hidden_size
+        if inner_hidden_size is None:
+            inner_hidden_size = 4 * hidden_size
+        self.inner_hidden_size = inner_hidden_size
+        self.dense_h_to_4h = init_method(
+            torch.nn.Linear,
+            self.hidden_size,
+            self.inner_hidden_size,
+            bias=bias,
+            dtype=params_dtype,
+        )
+        # Project back to h.
+        self.dense_4h_to_h = init_method(
+            torch.nn.Linear,
+            self.inner_hidden_size,
+            self.hidden_size,
+            bias=bias,
+            dtype=params_dtype,
+        )
+
+    def forward(self, hidden_states):
+        """
+        hidden_states: [seq_len, batch, hidden_size]
+        """
+
+        # [seq_len, batch, inner_hidden_size]
+        intermediate_parallel = self.dense_h_to_4h(hidden_states)
+
+        intermediate_parallel = self.activation_func(intermediate_parallel)
+
+        output = self.dense_4h_to_h(intermediate_parallel)
+
+        return output
+
+
+class GLMBlock(torch.nn.Module):
+    def __init__(
+            self,
+            hidden_size,
+            num_attention_heads,
+            layernorm_epsilon,
+            layer_id,
+            inner_hidden_size=None,
+            hidden_size_per_attention_head=None,
+            layernorm=LayerNorm,
+            use_bias=True,
+            params_dtype=torch.float,
+            num_layers=28,
+            position_encoding_2d=True,
+            empty_init=True
+    ):
+        super(GLMBlock, self).__init__()
+        # Set output layer initialization if not provided.
+
+        self.layer_id = layer_id
+
+        # Layernorm on the input data.
+        self.input_layernorm = layernorm(hidden_size, eps=layernorm_epsilon)
+
+        self.position_encoding_2d = position_encoding_2d
+
+        # Self attention.
+        self.attention = SelfAttention(
+            hidden_size,
+            num_attention_heads,
+            layer_id,
+            hidden_size_per_attention_head=hidden_size_per_attention_head,
+            bias=use_bias,
+            params_dtype=params_dtype,
+            position_encoding_2d=self.position_encoding_2d,
+            empty_init=empty_init
+        )
+
+        # Layernorm on the input data.
+        self.post_attention_layernorm = layernorm(hidden_size, eps=layernorm_epsilon)
+
+        self.num_layers = num_layers
+
+        # GLU
+        self.mlp = GLU(
+            hidden_size,
+            inner_hidden_size=inner_hidden_size,
+            bias=use_bias,
+            layer_id=layer_id,
+            params_dtype=params_dtype,
+            empty_init=empty_init
+        )
+
+    def forward(
+            self,
+            hidden_states: torch.Tensor,
+            position_ids,
+            attention_mask: torch.Tensor,
+            layer_id,
+            layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+            use_cache: bool = False,
+            output_attentions: bool = False,
+    ):
+        """
+        hidden_states: [seq_len, batch, hidden_size]
+        attention_mask: [(1, 1), seq_len, seq_len]
+        """
+
+        # Layer norm at the begining of the transformer layer.
+        # [seq_len, batch, hidden_size]
+        attention_input = self.input_layernorm(hidden_states)
+
+        # Self attention.
+        attention_outputs = self.attention(
+            attention_input,
+            position_ids,
+            attention_mask=attention_mask,
+            layer_id=layer_id,
+            layer_past=layer_past,
+            use_cache=use_cache,
+            output_attentions=output_attentions
+        )
+
+        attention_output = attention_outputs[0]
+
+        outputs = attention_outputs[1:]
+
+        # Residual connection.
+        alpha = (2 * self.num_layers) ** 0.5
+        hidden_states = attention_input * alpha + attention_output
+
+        mlp_input = self.post_attention_layernorm(hidden_states)
+
+        # MLP.
+        mlp_output = self.mlp(mlp_input)
+
+        # Second residual connection.
+        output = mlp_input * alpha + mlp_output
+
+        if use_cache:
+            outputs = (output,) + outputs
+        else:
+            outputs = (output,) + outputs[1:]
+
+        return outputs  # hidden_states, present, attentions
+
+
+class ChatGLMPreTrainedModel(PreTrainedModel):
+    """
+    An abstract class to handle weights initialization and
+    a simple interface for downloading and loading pretrained models.
+    """
+
+    is_parallelizable = False
+    supports_gradient_checkpointing = True
+    config_class = ChatGLMConfig
+    base_model_prefix = "transformer"
+    _no_split_modules = ["GLMBlock"]
+
+    def __init__(self, *inputs, **kwargs):
+        super().__init__(*inputs, **kwargs)
+
+    def _init_weights(self, module: nn.Module):
+        """Initialize the weights."""
+        return
+
+    def get_masks(self, input_ids, device):
+        batch_size, seq_length = input_ids.shape
+        context_lengths = [seq.tolist().index(self.config.bos_token_id) for seq in input_ids]
+        attention_mask = torch.ones((batch_size, seq_length, seq_length), device=device)
+        attention_mask.tril_()
+        for i, context_length in enumerate(context_lengths):
+            attention_mask[i, :, :context_length] = 1
+        attention_mask.unsqueeze_(1)
+        attention_mask = (attention_mask < 0.5).bool()
+
+        return attention_mask
+
+    def get_position_ids(self, input_ids, mask_positions, device, use_gmasks=None):
+        batch_size, seq_length = input_ids.shape
+        if use_gmasks is None:
+            use_gmasks = [False] * batch_size
+        context_lengths = [seq.tolist().index(self.config.bos_token_id) for seq in input_ids]
+        if self.position_encoding_2d:
+            position_ids = torch.arange(seq_length, dtype=torch.long, device=device).unsqueeze(0).repeat(batch_size, 1)
+            for i, context_length in enumerate(context_lengths):
+                position_ids[i, context_length:] = mask_positions[i]
+            block_position_ids = [torch.cat((
+                torch.zeros(context_length, dtype=torch.long, device=device),
+                torch.arange(seq_length - context_length, dtype=torch.long, device=device) + 1
+            )) for context_length in context_lengths]
+            block_position_ids = torch.stack(block_position_ids, dim=0)
+            position_ids = torch.stack((position_ids, block_position_ids), dim=1)
+        else:
+            position_ids = torch.arange(seq_length, dtype=torch.long, device=device).unsqueeze(0).repeat(batch_size, 1)
+            for i, context_length in enumerate(context_lengths):
+                if not use_gmasks[i]:
+                    position_ids[i, context_length:] = mask_positions[i]
+
+        return position_ids
+
+    def _set_gradient_checkpointing(self, module, value=False):
+        if isinstance(module, ChatGLMModel):
+            module.gradient_checkpointing = value
+
+
+CHATGLM_6B_START_DOCSTRING = r"""
+    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general
+    usage and behavior.
+
+    Parameters:
+        config ([`~ChatGLM6BConfig`]): Model configuration class with all the parameters of the model.
+            Initializing with a config file does not load the weights associated with the model, only the configuration.
+            Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+CHATGLM_6B_INPUTS_DOCSTRING = r"""
+    Args:
+        input_ids (`torch.LongTensor` of shape `({0})`):
+            Indices of input sequence tokens in the vocabulary.
+
+            Indices can be obtained using [`ChatGLM6BTokenizer`].
+            See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
+            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+
+            [What are attention masks?](../glossary#attention-mask)
+        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
+            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+
+            - 0 corresponds to a *sentence A* token,
+            - 1 corresponds to a *sentence B* token.
+
+            [What are token type IDs?](../glossary#token-type-ids)
+        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
+            Indices of positions of each input sequence tokens in the position embeddings.
+            Selected in the range `[0, config.max_position_embeddings - 1]`.
+
+            [What are position IDs?](../glossary#position-ids)
+        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
+            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
+
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+
+        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
+            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+            This is useful if you want more control over how to convert *input_ids* indices into associated vectors
+            than the model's internal embedding lookup matrix.
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+@add_start_docstrings(
+    "The bare ChatGLM-6B Model transformer outputting raw hidden-states without any specific head on top.",
+    CHATGLM_6B_START_DOCSTRING,
+)
+class ChatGLMModel(ChatGLMPreTrainedModel):
+    """
+
+    The model can behave as an encoder (with only self-attention) as well
+    as a decoder, in which case a layer of cross-attention is added between
+    the self-attention layers, following the architecture described in [Attention is
+    all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani,
+    Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
+
+    To behave as an decoder the model needs to be initialized with the
+    `is_decoder` argument of the configuration set to `True`.
+    To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder`
+    argument and `add_cross_attention` set to `True`; an
+    `encoder_hidden_states` is then expected as an input to the forward pass.
+    """
+
+    def __init__(self, config: ChatGLMConfig, empty_init=True):
+        super().__init__(config)
+        if empty_init:
+            init_method = skip_init
+        else:
+            init_method = default_init
+        # recording parameters
+        self.max_sequence_length = config.max_sequence_length
+        self.hidden_size = config.hidden_size
+        self.params_dtype = torch.half
+        self.num_attention_heads = config.num_attention_heads
+        self.vocab_size = config.vocab_size
+        self.num_layers = config.num_layers
+        self.layernorm_epsilon = config.layernorm_epsilon
+        self.inner_hidden_size = config.inner_hidden_size
+        self.hidden_size_per_attention_head = self.hidden_size // self.num_attention_heads
+        self.position_encoding_2d = config.position_encoding_2d
+        self.pre_seq_len = config.pre_seq_len
+        self.prefix_projection = config.prefix_projection
+
+        self.word_embeddings = init_method(
+            torch.nn.Embedding,
+            num_embeddings=self.vocab_size, embedding_dim=self.hidden_size,
+            dtype=self.params_dtype
+        )
+        self.gradient_checkpointing = False
+
+        def get_layer(layer_id):
+            return GLMBlock(
+                self.hidden_size,
+                self.num_attention_heads,
+                self.layernorm_epsilon,
+                layer_id,
+                inner_hidden_size=self.inner_hidden_size,
+                hidden_size_per_attention_head=self.hidden_size_per_attention_head,
+                layernorm=LayerNorm,
+                use_bias=True,
+                params_dtype=self.params_dtype,
+                position_encoding_2d=self.position_encoding_2d,
+                empty_init=empty_init
+            )
+
+        self.layers = torch.nn.ModuleList(
+            [get_layer(layer_id) for layer_id in range(self.num_layers)]
+        )
+
+        # Final layer norm before output.
+        self.final_layernorm = LayerNorm(self.hidden_size, eps=self.layernorm_epsilon)
+
+        if self.pre_seq_len is not None:
+            for param in self.parameters():
+                param.requires_grad = False
+            self.prefix_tokens = torch.arange(self.pre_seq_len).long()
+            self.prefix_encoder = PrefixEncoder(config)
+            self.dropout = torch.nn.Dropout(0.1)
+
+            # total_params = sum(p.numel() for p in self.parameters())
+            # trainable_params = sum(p.numel() for p in self.parameters() if p.requires_grad)
+            # print("Using p-tuning v2: # trainable_params = {} / {}".format(trainable_params, total_params))
+
+    def get_input_embeddings(self):
+        return self.word_embeddings
+
+    def set_input_embeddings(self, new_embeddings: torch.Tensor):
+        self.word_embeddings = new_embeddings
+
+    def get_prompt(self, batch_size, device, dtype=torch.half):
+        prefix_tokens = self.prefix_tokens.unsqueeze(0).expand(batch_size, -1).to(device)
+        past_key_values = self.prefix_encoder(prefix_tokens).type(dtype)
+        past_key_values = past_key_values.view(
+            batch_size,
+            self.pre_seq_len,
+            self.num_layers * 2,
+            self.num_attention_heads,
+            self.hidden_size // self.num_attention_heads
+        )
+        # seq_len, b, nh, hidden_size
+        past_key_values = self.dropout(past_key_values)
+        past_key_values = past_key_values.permute([2, 1, 0, 3, 4]).split(2)
+        # past_key_values = [(v[0], v[1]) for v in past_key_values]
+        return past_key_values
+
+    @add_start_docstrings_to_model_forward(CHATGLM_6B_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
+    @add_code_sample_docstrings(
+        checkpoint=_CHECKPOINT_FOR_DOC,
+        output_type=BaseModelOutputWithPastAndCrossAttentions,
+        config_class=_CONFIG_FOR_DOC,
+    )
+    def forward(
+            self,
+            input_ids: Optional[torch.LongTensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,
+            inputs_embeds: Optional[torch.LongTensor] = None,
+            use_cache: Optional[bool] = None,
+            output_attentions: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+    ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPast]:
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning_once(
+                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                )
+                use_cache = False
+
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            batch_size, seq_length = input_ids.shape[:2]
+        elif inputs_embeds is not None:
+            batch_size, seq_length = inputs_embeds.shape[:2]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        if inputs_embeds is None:
+            inputs_embeds = self.word_embeddings(input_ids)
+
+        if past_key_values is None:
+            if self.pre_seq_len is not None:
+                past_key_values = self.get_prompt(batch_size=input_ids.shape[0], device=input_ids.device,
+                                                  dtype=inputs_embeds.dtype)
+            else:
+                past_key_values = tuple([None] * len(self.layers))
+
+            if attention_mask is None:
+                attention_mask = self.get_masks(
+                    input_ids,
+                    device=input_ids.device
+                )
+
+
+            if position_ids is None:
+                MASK, gMASK = self.config.mask_token_id, self.config.gmask_token_id
+                seqs = input_ids.tolist()
+
+                mask_positions, use_gmasks = [], []
+                for seq in seqs:
+                    mask_token = gMASK if gMASK in seq else MASK
+                    use_gmask = mask_token == gMASK
+                    mask_positions.append(seq.index(mask_token))
+                    use_gmasks.append(use_gmask)
+
+                position_ids = self.get_position_ids(
+                    input_ids,
+                    mask_positions=mask_positions,
+                    device=input_ids.device,
+                    use_gmasks=use_gmasks
+                )
+
+        if self.pre_seq_len is not None and attention_mask is not None:
+            prefix_attention_mask = torch.ones(batch_size, 1, input_ids.size(-1), self.pre_seq_len).to(
+                attention_mask.device)
+            prefix_attention_mask = (prefix_attention_mask < 0.5).bool()
+            attention_mask = torch.cat((prefix_attention_mask, attention_mask), dim=3)
+
+        # [seq_len, batch, hidden_size]
+        hidden_states = inputs_embeds.transpose(0, 1)
+
+        presents = () if use_cache else None
+        all_self_attentions = () if output_attentions else None
+        all_hidden_states = () if output_hidden_states else None
+
+        if attention_mask is None:
+            attention_mask = torch.zeros(1, 1, device=input_ids.device).bool()
+        else:
+            attention_mask = attention_mask.to(hidden_states.device)
+
+        for i, layer in enumerate(self.layers):
+
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+            layer_past = past_key_values[i]
+
+            if self.gradient_checkpointing and self.training:
+                layer_ret = torch.utils.checkpoint.checkpoint(
+                    layer,
+                    hidden_states,
+                    position_ids,
+                    attention_mask,
+                    torch.tensor(i),
+                    layer_past,
+                    use_cache,
+                    output_attentions
+                )
+            else:
+                layer_ret = layer(
+                    hidden_states,
+                    position_ids=position_ids,
+                    attention_mask=attention_mask,
+                    layer_id=torch.tensor(i),
+                    layer_past=layer_past,
+                    use_cache=use_cache,
+                    output_attentions=output_attentions
+                )
+
+            hidden_states = layer_ret[0]
+
+            if use_cache:
+                presents = presents + (layer_ret[1],)
+
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (layer_ret[2 if use_cache else 1],)
+
+        # Final layer norm.
+        hidden_states = self.final_layernorm(hidden_states)
+
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)
+
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=presents,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+        )
+
+
+class ChatGLMForConditionalGeneration(ChatGLMPreTrainedModel):
+    def __init__(self, config: ChatGLMConfig, empty_init=True):
+        super().__init__(config)
+        if empty_init:
+            init_method = skip_init
+        else:
+            init_method = default_init
+
+        # self.hidden_size = config.hidden_size
+        # self.params_dtype = torch.half
+        # self.vocab_size = config.vocab_size
+        self.max_sequence_length = config.max_sequence_length
+
+        self.position_encoding_2d = config.position_encoding_2d
+
+        self.transformer = ChatGLMModel(config, empty_init=empty_init)
+
+        self.lm_head = init_method(
+            nn.Linear,
+            config.hidden_size,
+            config.vocab_size,
+            bias=False,
+            dtype=torch.half
+        )
+
+        self.config = config
+
+        self.quantized = False
+
+        if self.config.quantization_bit:
+            self.quantize(self.config.quantization_bit, empty_init=True)
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def _update_model_kwargs_for_generation(
+        self,
+        outputs: ModelOutput,
+        model_kwargs: Dict[str, Any],
+        is_encoder_decoder: bool = False,
+        standardize_cache_format: bool = False,
+    ) -> Dict[str, Any]:
+        # update past_key_values
+        model_kwargs["past_key_values"] = self._extract_past_from_model_output(
+            outputs, standardize_cache_format=standardize_cache_format
+        )
+
+        # update attention mask
+        if "attention_mask" in model_kwargs:
+            attention_mask = model_kwargs["attention_mask"]
+            if attention_mask is not None and attention_mask.dtype == torch.bool:
+                attention_mask = torch.cat(
+                    [attention_mask, attention_mask.new_ones((*attention_mask.shape[:3], 1))], dim=3)
+                new_attention_mask = attention_mask[:, :, -1:].clone()
+                new_attention_mask[..., -1] = False
+                model_kwargs["attention_mask"] = torch.cat(
+                    [attention_mask, new_attention_mask], dim=2
+                )
+
+        # update position ids
+        if "position_ids" in model_kwargs:
+            position_ids = model_kwargs["position_ids"]
+            new_position_id = position_ids[..., -1:].clone()
+            new_position_id[:, 1, :] += 1
+            model_kwargs["position_ids"] = torch.cat(
+                [position_ids, new_position_id], dim=-1
+            )
+
+        return model_kwargs
+
+    def prepare_inputs_for_generation(
+            self,
+            input_ids: torch.LongTensor,
+            past: Optional[torch.Tensor] = None,
+            past_key_values: Optional[torch.Tensor] = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.Tensor] = None,
+            **kwargs
+    ) -> dict:
+        batch_size, seq_length = input_ids.shape
+        MASK, gMASK = self.config.mask_token_id, self.config.gmask_token_id
+        seqs = input_ids.tolist()
+        mask_positions, use_gmasks = [], []
+        for seq in seqs:
+            mask_token = gMASK if gMASK in seq else MASK
+            use_gmask = mask_token == gMASK
+            mask_positions.append(seq.index(mask_token))
+            use_gmasks.append(use_gmask)
+
+        # only last token for input_ids if past is not None
+        if past is not None or past_key_values is not None:
+            last_token = input_ids[:, -1].unsqueeze(-1)
+            if attention_mask is not None and attention_mask.dtype == torch.bool:
+                attention_mask = attention_mask[:, :, -1:]
+            else:
+                attention_mask = None
+            if position_ids is not None:
+                position_ids = position_ids[..., -1:]
+            else:
+                context_lengths = [seq.index(self.config.bos_token_id) for seq in seqs]
+                if self.position_encoding_2d:
+                    position_ids = torch.tensor(
+                        [[mask_position, seq_length - context_length] for mask_position, context_length in
+                         zip(mask_positions, context_lengths)], dtype=torch.long, device=input_ids.device).unsqueeze(-1)
+                else:
+                    position_ids = torch.tensor([mask_position for mask_position in mask_positions], dtype=torch.long,
+                                                device=input_ids.device).unsqueeze(-1)
+
+            if past is None:
+                past = past_key_values
+            return {
+                "input_ids": last_token,
+                "past_key_values": past,
+                "position_ids": position_ids,
+                "attention_mask": attention_mask
+            }
+        else:
+            if attention_mask is not None and attention_mask.dtype != torch.bool:
+                logger.warning_once(f"The dtype of attention mask ({attention_mask.dtype}) is not bool")
+                attention_mask = None
+            if attention_mask is None:
+                attention_mask = self.get_masks(
+                    input_ids,
+                    device=input_ids.device
+                )
+            if position_ids is None:
+                position_ids = self.get_position_ids(
+                    input_ids,
+                    device=input_ids.device,
+                    mask_positions=mask_positions,
+                    use_gmasks=use_gmasks
+                )
+
+            return {
+                "input_ids": input_ids,
+                "past_key_values": past,
+                "position_ids": position_ids,
+                "attention_mask": attention_mask
+            }
+
+    def forward(
+            self,
+            input_ids: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.Tensor] = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            past_key_values: Optional[Tuple[torch.FloatTensor]] = None,
+            inputs_embeds: Optional[torch.Tensor] = None,
+            labels: Optional[torch.Tensor] = None,
+            use_cache: Optional[bool] = None,
+            output_attentions: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+    ):
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        transformer_outputs = self.transformer(
+            input_ids=input_ids,
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = transformer_outputs[0]
+
+        lm_logits = self.lm_head(hidden_states).permute(1, 0, 2).contiguous()
+
+        loss = None
+        if labels is not None:
+            lm_logits = lm_logits.to(torch.float32)
+
+            # Shift so that tokens < n predict n
+            shift_logits = lm_logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss(ignore_index=-100)
+            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+
+            lm_logits = lm_logits.to(hidden_states.dtype)
+            loss = loss.to(hidden_states.dtype)
+
+        if not return_dict:
+            output = (lm_logits,) + transformer_outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=lm_logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )
+
+    @staticmethod
+    def _reorder_cache(
+            past: Tuple[Tuple[torch.Tensor, torch.Tensor], ...], beam_idx: torch.LongTensor
+    ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], ...]:
+        """
+        This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or
+        [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
+        beam_idx at every generation step.
+
+        Output shares the same memory storage as `past`.
+        """
+        return tuple(
+            (
+                layer_past[0].index_select(1, beam_idx.to(layer_past[0].device)),
+                layer_past[1].index_select(1, beam_idx.to(layer_past[1].device)),
+            )
+            for layer_past in past
+        )
+
+    def process_response(self, response):
+        response = response.strip()
+        response = response.replace("[[训练时间]]", "2023年")
+        punkts = [
+            [",", "，"],
+            ["!", "！"],
+            [":", "："],
+            [";", "；"],
+            ["\?", "？"],
+        ]
+        for item in punkts:
+            response = re.sub(r"([\u4e00-\u9fff])%s" % item[0], r"\1%s" % item[1], response)
+            response = re.sub(r"%s([\u4e00-\u9fff])" % item[0], r"%s\1" % item[1], response)
+        return response
+
+    @torch.no_grad()
+    def chat(self, tokenizer, query: str, history: List[Tuple[str, str]] = None, max_length: int = 2048, num_beams=1,
+             do_sample=True, top_p=0.7, temperature=0.95, logits_processor=None, **kwargs):
+        if history is None:
+            history = []
+        if logits_processor is None:
+            logits_processor = LogitsProcessorList()
+        logits_processor.append(InvalidScoreLogitsProcessor())
+        gen_kwargs = {"max_length": max_length, "num_beams": num_beams, "do_sample": do_sample, "top_p": top_p,
+                      "temperature": temperature, "logits_processor": logits_processor, **kwargs}
+        if not history:
+            prompt = query
+        else:
+            prompt = ""
+            for i, (old_query, response) in enumerate(history):
+                prompt += "[Round {}]\n问：{}\n答：{}\n".format(i, old_query, response)
+            prompt += "[Round {}]\n问：{}\n答：".format(len(history), query)
+        inputs = tokenizer([prompt], return_tensors="pt")
+        inputs = inputs.to(self.device)
+        outputs = self.generate(**inputs, **gen_kwargs)
+        outputs = outputs.tolist()[0][len(inputs["input_ids"][0]):]
+        response = tokenizer.decode(outputs)
+        response = self.process_response(response)
+        history = history + [(query, response)]
+        return response, history
+
+    @torch.no_grad()
+    def stream_chat(self, tokenizer, query: str, history: List[Tuple[str, str]] = None, max_length: int = 2048,
+                    do_sample=True, top_p=0.7, temperature=0.95, logits_processor=None, **kwargs):
+        if history is None:
+            history = []
+        if logits_processor is None:
+            logits_processor = LogitsProcessorList()
+        logits_processor.append(InvalidScoreLogitsProcessor())
+        gen_kwargs = {"max_length": max_length, "do_sample": do_sample, "top_p": top_p,
+                      "temperature": temperature, "logits_processor": logits_processor, **kwargs}
+        if not history:
+            prompt = query
+        else:
+            prompt = ""
+            for i, (old_query, response) in enumerate(history):
+                prompt += "[Round {}]\n问：{}\n答：{}\n".format(i, old_query, response)
+            prompt += "[Round {}]\n问：{}\n答：".format(len(history), query)
+        inputs = tokenizer([prompt], return_tensors="pt")
+        inputs = inputs.to(self.device)
+        for outputs in self.stream_generate(**inputs, **gen_kwargs):
+            outputs = outputs.tolist()[0][len(inputs["input_ids"][0]):]
+            response = tokenizer.decode(outputs)
+            response = self.process_response(response)
+            new_history = history + [(query, response)]
+            yield response, new_history
+
+    @torch.no_grad()
+    def stream_generate(
+            self,
+            input_ids,
+            generation_config: Optional[GenerationConfig] = None,
+            logits_processor: Optional[LogitsProcessorList] = None,
+            stopping_criteria: Optional[StoppingCriteriaList] = None,
+            prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], List[int]]] = None,
+            **kwargs,
+    ):
+        batch_size, input_ids_seq_length = input_ids.shape[0], input_ids.shape[-1]
+
+        if generation_config is None:
+            generation_config = self.generation_config
+        generation_config = copy.deepcopy(generation_config)
+        model_kwargs = generation_config.update(**kwargs)
+        bos_token_id, eos_token_id = generation_config.bos_token_id, generation_config.eos_token_id
+
+        if isinstance(eos_token_id, int):
+            eos_token_id = [eos_token_id]
+
+        has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None
+        if has_default_max_length and generation_config.max_new_tokens is None:
+            warnings.warn(
+                f"Using `max_length`'s default ({generation_config.max_length}) to control the generation length. "
+                "This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we"
+                " recommend using `max_new_tokens` to control the maximum length of the generation.",
+                UserWarning,
+            )
+        elif generation_config.max_new_tokens is not None:
+            generation_config.max_length = generation_config.max_new_tokens + input_ids_seq_length
+            if not has_default_max_length:
+                logger.warn(
+                    f"Both `max_new_tokens` (={generation_config.max_new_tokens}) and `max_length`(="
+                    f"{generation_config.max_length}) seem to have been set. `max_new_tokens` will take precedence. "
+                    "Please refer to the documentation for more information. "
+                    "(https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)",
+                    UserWarning,
+                )
+
+        if input_ids_seq_length >= generation_config.max_length:
+            input_ids_string = "decoder_input_ids" if self.config.is_encoder_decoder else "input_ids"
+            logger.warning(
+                f"Input length of {input_ids_string} is {input_ids_seq_length}, but `max_length` is set to"
+                f" {generation_config.max_length}. This can lead to unexpected behavior. You should consider"
+                " increasing `max_new_tokens`."
+            )
+
+        # 2. Set generation parameters if not already defined
+        logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
+        stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()
+
+        logits_processor = self._get_logits_processor(
+            generation_config=generation_config,
+            input_ids_seq_length=input_ids_seq_length,
+            encoder_input_ids=input_ids,
+            prefix_allowed_tokens_fn=prefix_allowed_tokens_fn,
+            logits_processor=logits_processor,
+        )
+
+        stopping_criteria = self._get_stopping_criteria(
+            generation_config=generation_config, stopping_criteria=stopping_criteria
+        )
+        logits_warper = self._get_logits_warper(generation_config)
+
+        unfinished_sequences = input_ids.new(input_ids.shape[0]).fill_(1)
+        scores = None
+        while True:
+            model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
+            # forward pass to get next token
+            outputs = self(
+                **model_inputs,
+                return_dict=True,
+                output_attentions=False,
+                output_hidden_states=False,
+            )
+
+            next_token_logits = outputs.logits[:, -1, :]
+
+            # pre-process distribution
+            next_token_scores = logits_processor(input_ids, next_token_logits)
+            next_token_scores = logits_warper(input_ids, next_token_scores)
+
+            # sample
+            probs = nn.functional.softmax(next_token_scores, dim=-1)
+            if generation_config.do_sample:
+                next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1)
+            else:
+                next_tokens = torch.argmax(probs, dim=-1)
+
+            # update generated ids, model inputs, and length for next step
+            input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1)
+            model_kwargs = self._update_model_kwargs_for_generation(
+                outputs, model_kwargs, is_encoder_decoder=self.config.is_encoder_decoder
+            )
+            unfinished_sequences = unfinished_sequences.mul((sum(next_tokens != i for i in eos_token_id)).long())
+
+            # stop when each sentence is finished, or if we exceed the maximum length
+            if unfinished_sequences.max() == 0 or stopping_criteria(input_ids, scores):
+                break
+            yield input_ids
+
+    def quantize(self, bits: int, empty_init=False, **kwargs):
+        if bits == 0:
+            return
+
+        from .quantization import quantize
+
+        if self.quantized:
+            logger.info("Already quantized.")
+            return self
+
+        self.quantized = True
+
+        self.config.quantization_bit = bits
+
+        self.transformer = quantize(self.transformer, bits, empty_init=empty_init, **kwargs)
+        return self

pytorch_model-00001-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f84fe795f562e0f9be32feecda08dea1fa4c6f046cdead3f6ae13fdf6f27033
+size 537

pytorch_model-00002-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e9140cfa89a930dfa52788e3245bf43820cc47f48e2712e307361e0892e1bd7
+size 1069286378

pytorch_model-00003-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9bd1363d3c5877b2f887bc9e3ffed8d791166b63a423b8391c4a1ac624d4baf
+size 939814661

pytorch_model-00004-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:423027474027c43a23879bcf5f83596077941c39d384d04512c3140ee7537416
+size 939779463

pytorch_model-00005-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d88a8e76b606ff686522feea633993a65a3d3a1fa18b48d9320ca14d9c6b192
+size 939772285

pytorch_model-00006-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d0920fb8aee64f0222ce8e1ce0a58198b26fb0e1aee05202060df771a4e4b5c
+size 939797199

pytorch_model-00007-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30d73ddef20d36299ee8e92f7df14fb8c79696884b18b4ffdafc4df4e5f466e8
+size 939779527

pytorch_model-00008-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70b42e48761fa92e8bf3edee0e33504bc497236935edb7a83f2b5662c4e0ee7f
+size 939772349

pytorch_model-00009-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7c96fe1f921708bbd216a57b0f09ffaf7aa28cd53be317670d3938fece4ae59
+size 939797199

pytorch_model-00010-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0b6c3976aab6621a305e36267216423ae64f21c156c86b44d896d9fcf0b0beb
+size 939779527

pytorch_model-00011-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49c1b9f22528ee7c9570b445411f6d848eaf1af5ac6cbef20b94a817b863ec49
+size 939772349

pytorch_model-00012-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79f668c64fce5cd808c0d56ab071d75fb4e5a79fb258a59cbcf93dfd6308f933
+size 939797199

pytorch_model-00013-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cfc8f014ee26a84336df8077c48d4371690701732111cd4de3f085e3e8e9e0f
+size 939779527

pytorch_model-00014-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aba46ce9ad2ff05fda5f5a06fad6b7e843cd11a8f906d408275783e12a374640
+size 939771947

pytorch_model.bin.index.json

@@ -0,0 +1,375 @@
+{
+  "metadata": {
+    "total_size": 12346574592
+  },
+  "weight_map": {
+    "lm_head.weight": "pytorch_model-00002-of-00014.bin",
+    "transformer.final_layernorm.bias": "pytorch_model-00014-of-00014.bin",
+    "transformer.final_layernorm.weight": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.0.attention.dense.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.0.attention.dense.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.0.attention.query_key_value.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.0.attention.query_key_value.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.0.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.0.input_layernorm.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.0.input_layernorm.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.0.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.0.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.0.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.0.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.0.post_attention_layernorm.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.0.post_attention_layernorm.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.1.attention.dense.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.1.attention.dense.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.1.attention.query_key_value.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.1.attention.query_key_value.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.1.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.1.input_layernorm.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.1.input_layernorm.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.1.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.1.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.1.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.1.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.1.post_attention_layernorm.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.1.post_attention_layernorm.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.10.attention.dense.bias": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.10.attention.dense.weight": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.10.attention.query_key_value.bias": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.10.attention.query_key_value.weight": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.10.attention.rotary_emb.inv_freq": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.10.input_layernorm.bias": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.10.input_layernorm.weight": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.10.mlp.dense_4h_to_h.bias": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.10.mlp.dense_4h_to_h.weight": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.10.mlp.dense_h_to_4h.bias": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.10.mlp.dense_h_to_4h.weight": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.10.post_attention_layernorm.bias": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.10.post_attention_layernorm.weight": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.11.attention.dense.bias": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.11.attention.dense.weight": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.11.attention.query_key_value.bias": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.11.attention.query_key_value.weight": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.11.attention.rotary_emb.inv_freq": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.11.input_layernorm.bias": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.11.input_layernorm.weight": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.11.mlp.dense_4h_to_h.bias": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.11.mlp.dense_4h_to_h.weight": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.11.mlp.dense_h_to_4h.bias": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.11.mlp.dense_h_to_4h.weight": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.11.post_attention_layernorm.bias": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.11.post_attention_layernorm.weight": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.12.attention.dense.bias": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.12.attention.dense.weight": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.12.attention.query_key_value.bias": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.12.attention.query_key_value.weight": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.12.attention.rotary_emb.inv_freq": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.12.input_layernorm.bias": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.12.input_layernorm.weight": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.12.mlp.dense_4h_to_h.bias": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.12.mlp.dense_4h_to_h.weight": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.12.mlp.dense_h_to_4h.bias": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.12.mlp.dense_h_to_4h.weight": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.12.post_attention_layernorm.bias": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.12.post_attention_layernorm.weight": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.13.attention.dense.bias": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.13.attention.dense.weight": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.13.attention.query_key_value.bias": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.13.attention.query_key_value.weight": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.13.attention.rotary_emb.inv_freq": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.13.input_layernorm.bias": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.13.input_layernorm.weight": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.13.mlp.dense_4h_to_h.bias": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.13.mlp.dense_4h_to_h.weight": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.13.mlp.dense_h_to_4h.bias": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.13.mlp.dense_h_to_4h.weight": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.13.post_attention_layernorm.bias": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.13.post_attention_layernorm.weight": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.14.attention.dense.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.14.attention.dense.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.14.attention.query_key_value.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.14.attention.query_key_value.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.14.attention.rotary_emb.inv_freq": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.14.input_layernorm.bias": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.14.input_layernorm.weight": "pytorch_model-00008-of-00014.bin",
+    "transformer.layers.14.mlp.dense_4h_to_h.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.14.mlp.dense_4h_to_h.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.14.mlp.dense_h_to_4h.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.14.mlp.dense_h_to_4h.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.14.post_attention_layernorm.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.14.post_attention_layernorm.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.15.attention.dense.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.15.attention.dense.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.15.attention.query_key_value.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.15.attention.query_key_value.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.15.attention.rotary_emb.inv_freq": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.15.input_layernorm.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.15.input_layernorm.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.15.mlp.dense_4h_to_h.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.15.mlp.dense_4h_to_h.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.15.mlp.dense_h_to_4h.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.15.mlp.dense_h_to_4h.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.15.post_attention_layernorm.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.15.post_attention_layernorm.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.16.attention.dense.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.16.attention.dense.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.16.attention.query_key_value.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.16.attention.query_key_value.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.16.attention.rotary_emb.inv_freq": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.16.input_layernorm.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.16.input_layernorm.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.16.mlp.dense_4h_to_h.bias": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.16.mlp.dense_4h_to_h.weight": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.16.mlp.dense_h_to_4h.bias": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.16.mlp.dense_h_to_4h.weight": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.16.post_attention_layernorm.bias": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.16.post_attention_layernorm.weight": "pytorch_model-00009-of-00014.bin",
+    "transformer.layers.17.attention.dense.bias": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.17.attention.dense.weight": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.17.attention.query_key_value.bias": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.17.attention.query_key_value.weight": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.17.attention.rotary_emb.inv_freq": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.17.input_layernorm.bias": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.17.input_layernorm.weight": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.17.mlp.dense_4h_to_h.bias": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.17.mlp.dense_4h_to_h.weight": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.17.mlp.dense_h_to_4h.bias": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.17.mlp.dense_h_to_4h.weight": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.17.post_attention_layernorm.bias": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.17.post_attention_layernorm.weight": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.18.attention.dense.bias": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.18.attention.dense.weight": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.18.attention.query_key_value.bias": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.18.attention.query_key_value.weight": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.18.attention.rotary_emb.inv_freq": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.18.input_layernorm.bias": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.18.input_layernorm.weight": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.18.mlp.dense_4h_to_h.bias": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.18.mlp.dense_4h_to_h.weight": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.18.mlp.dense_h_to_4h.bias": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.18.mlp.dense_h_to_4h.weight": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.18.post_attention_layernorm.bias": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.18.post_attention_layernorm.weight": "pytorch_model-00010-of-00014.bin",
+    "transformer.layers.19.attention.dense.bias": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.19.attention.dense.weight": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.19.attention.query_key_value.bias": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.19.attention.query_key_value.weight": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.19.attention.rotary_emb.inv_freq": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.19.input_layernorm.bias": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.19.input_layernorm.weight": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.19.mlp.dense_4h_to_h.bias": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.19.mlp.dense_4h_to_h.weight": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.19.mlp.dense_h_to_4h.bias": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.19.mlp.dense_h_to_4h.weight": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.19.post_attention_layernorm.bias": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.19.post_attention_layernorm.weight": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.2.attention.dense.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.2.attention.dense.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.2.attention.query_key_value.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.2.attention.query_key_value.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.2.attention.rotary_emb.inv_freq": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.2.input_layernorm.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.2.input_layernorm.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.2.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.2.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.2.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.2.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.2.post_attention_layernorm.bias": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.2.post_attention_layernorm.weight": "pytorch_model-00003-of-00014.bin",
+    "transformer.layers.20.attention.dense.bias": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.20.attention.dense.weight": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.20.attention.query_key_value.bias": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.20.attention.query_key_value.weight": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.20.attention.rotary_emb.inv_freq": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.20.input_layernorm.bias": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.20.input_layernorm.weight": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.20.mlp.dense_4h_to_h.bias": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.20.mlp.dense_4h_to_h.weight": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.20.mlp.dense_h_to_4h.bias": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.20.mlp.dense_h_to_4h.weight": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.20.post_attention_layernorm.bias": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.20.post_attention_layernorm.weight": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.21.attention.dense.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.21.attention.dense.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.21.attention.query_key_value.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.21.attention.query_key_value.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.21.attention.rotary_emb.inv_freq": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.21.input_layernorm.bias": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.21.input_layernorm.weight": "pytorch_model-00011-of-00014.bin",
+    "transformer.layers.21.mlp.dense_4h_to_h.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.21.mlp.dense_4h_to_h.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.21.mlp.dense_h_to_4h.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.21.mlp.dense_h_to_4h.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.21.post_attention_layernorm.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.21.post_attention_layernorm.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.22.attention.dense.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.22.attention.dense.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.22.attention.query_key_value.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.22.attention.query_key_value.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.22.attention.rotary_emb.inv_freq": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.22.input_layernorm.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.22.input_layernorm.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.22.mlp.dense_4h_to_h.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.22.mlp.dense_4h_to_h.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.22.mlp.dense_h_to_4h.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.22.mlp.dense_h_to_4h.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.22.post_attention_layernorm.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.22.post_attention_layernorm.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.23.attention.dense.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.23.attention.dense.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.23.attention.query_key_value.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.23.attention.query_key_value.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.23.attention.rotary_emb.inv_freq": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.23.input_layernorm.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.23.input_layernorm.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.23.mlp.dense_4h_to_h.bias": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.23.mlp.dense_4h_to_h.weight": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.23.mlp.dense_h_to_4h.bias": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.23.mlp.dense_h_to_4h.weight": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.23.post_attention_layernorm.bias": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.23.post_attention_layernorm.weight": "pytorch_model-00012-of-00014.bin",
+    "transformer.layers.24.attention.dense.bias": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.24.attention.dense.weight": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.24.attention.query_key_value.bias": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.24.attention.query_key_value.weight": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.24.attention.rotary_emb.inv_freq": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.24.input_layernorm.bias": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.24.input_layernorm.weight": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.24.mlp.dense_4h_to_h.bias": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.24.mlp.dense_4h_to_h.weight": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.24.mlp.dense_h_to_4h.bias": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.24.mlp.dense_h_to_4h.weight": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.24.post_attention_layernorm.bias": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.24.post_attention_layernorm.weight": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.25.attention.dense.bias": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.25.attention.dense.weight": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.25.attention.query_key_value.bias": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.25.attention.query_key_value.weight": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.25.attention.rotary_emb.inv_freq": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.25.input_layernorm.bias": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.25.input_layernorm.weight": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.25.mlp.dense_4h_to_h.bias": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.25.mlp.dense_4h_to_h.weight": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.25.mlp.dense_h_to_4h.bias": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.25.mlp.dense_h_to_4h.weight": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.25.post_attention_layernorm.bias": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.25.post_attention_layernorm.weight": "pytorch_model-00013-of-00014.bin",
+    "transformer.layers.26.attention.dense.bias": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.26.attention.dense.weight": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.26.attention.query_key_value.bias": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.26.attention.query_key_value.weight": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.26.attention.rotary_emb.inv_freq": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.26.input_layernorm.bias": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.26.input_layernorm.weight": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.26.mlp.dense_4h_to_h.bias": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.26.mlp.dense_4h_to_h.weight": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.26.mlp.dense_h_to_4h.bias": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.26.mlp.dense_h_to_4h.weight": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.26.post_attention_layernorm.bias": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.26.post_attention_layernorm.weight": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.27.attention.dense.bias": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.27.attention.dense.weight": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.27.attention.query_key_value.bias": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.27.attention.query_key_value.weight": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.27.attention.rotary_emb.inv_freq": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.27.input_layernorm.bias": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.27.input_layernorm.weight": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.27.mlp.dense_4h_to_h.bias": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.27.mlp.dense_4h_to_h.weight": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.27.mlp.dense_h_to_4h.bias": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.27.mlp.dense_h_to_4h.weight": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.27.post_attention_layernorm.bias": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.27.post_attention_layernorm.weight": "pytorch_model-00014-of-00014.bin",
+    "transformer.layers.3.attention.dense.bias": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.3.attention.dense.weight": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.3.attention.query_key_value.bias": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.3.attention.query_key_value.weight": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.3.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.3.input_layernorm.bias": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.3.input_layernorm.weight": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.3.mlp.dense_4h_to_h.bias": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.3.mlp.dense_4h_to_h.weight": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.3.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.3.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.3.post_attention_layernorm.bias": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.3.post_attention_layernorm.weight": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.4.attention.dense.bias": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.4.attention.dense.weight": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.4.attention.query_key_value.bias": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.4.attention.query_key_value.weight": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.4.attention.rotary_emb.inv_freq": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.4.input_layernorm.bias": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.4.input_layernorm.weight": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.4.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.4.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.4.mlp.dense_h_to_4h.bias": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.4.mlp.dense_h_to_4h.weight": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.4.post_attention_layernorm.bias": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.4.post_attention_layernorm.weight": "pytorch_model-00004-of-00014.bin",
+    "transformer.layers.5.attention.dense.bias": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.5.attention.dense.weight": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.5.attention.query_key_value.bias": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.5.attention.query_key_value.weight": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.5.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.5.input_layernorm.bias": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.5.input_layernorm.weight": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.5.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.5.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.5.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.5.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.5.post_attention_layernorm.bias": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.5.post_attention_layernorm.weight": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.6.attention.dense.bias": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.6.attention.dense.weight": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.6.attention.query_key_value.bias": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.6.attention.query_key_value.weight": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.6.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.6.input_layernorm.bias": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.6.input_layernorm.weight": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.6.mlp.dense_4h_to_h.bias": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.6.mlp.dense_4h_to_h.weight": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.6.mlp.dense_h_to_4h.bias": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.6.mlp.dense_h_to_4h.weight": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.6.post_attention_layernorm.bias": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.6.post_attention_layernorm.weight": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.7.attention.dense.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.7.attention.dense.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.7.attention.query_key_value.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.7.attention.query_key_value.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.7.attention.rotary_emb.inv_freq": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.7.input_layernorm.bias": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.7.input_layernorm.weight": "pytorch_model-00005-of-00014.bin",
+    "transformer.layers.7.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.7.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.7.mlp.dense_h_to_4h.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.7.mlp.dense_h_to_4h.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.7.post_attention_layernorm.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.7.post_attention_layernorm.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.8.attention.dense.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.8.attention.dense.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.8.attention.query_key_value.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.8.attention.query_key_value.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.8.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.8.input_layernorm.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.8.input_layernorm.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.8.mlp.dense_4h_to_h.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.8.mlp.dense_4h_to_h.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.8.mlp.dense_h_to_4h.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.8.mlp.dense_h_to_4h.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.8.post_attention_layernorm.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.8.post_attention_layernorm.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.9.attention.dense.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.9.attention.dense.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.9.attention.query_key_value.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.9.attention.query_key_value.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.9.attention.rotary_emb.inv_freq": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.9.input_layernorm.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.9.input_layernorm.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.9.mlp.dense_4h_to_h.bias": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.9.mlp.dense_4h_to_h.weight": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.9.mlp.dense_h_to_4h.bias": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.9.mlp.dense_h_to_4h.weight": "pytorch_model-00007-of-00014.bin",
+    "transformer.layers.9.post_attention_layernorm.bias": "pytorch_model-00006-of-00014.bin",
+    "transformer.layers.9.post_attention_layernorm.weight": "pytorch_model-00006-of-00014.bin",
+    "transformer.word_embeddings.weight": "pytorch_model-00002-of-00014.bin"
+  }
+}

quantization.py

@@ -0,0 +1,201 @@
+from torch.nn import Linear
+from torch.nn.parameter import Parameter
+
+import bz2
+import torch
+import base64
+import ctypes
+from transformers.utils import logging
+
+from typing import List
+from functools import partial
+
+logger = logging.get_logger(__name__)
+
+try:
+    from cpm_kernels.kernels.base import LazyKernelCModule, KernelFunction, round_up
+
+    class Kernel:
+        def __init__(self, code: bytes, function_names: List[str]):
+            self.code = code
+            self._function_names = function_names
+            self._cmodule = LazyKernelCModule(self.code)
+
+            for name in self._function_names:
+                setattr(self, name, KernelFunction(self._cmodule, name))
+
+    quantization_code = "$QlpoOTFBWSZTWU9yuJUAQHN//////////f/n/8/n///n//bt4dTidcVx8X3V9FV/92/v4B7/AD5FBQFAAAChSgKpFCFAFVSigUAAAEKhSgUUqgFBKigqVREQAABQBQIANDTTIGI00BkZBkNGE0A0BkBkGQGRkaNAaAGQNBoGgDIAAYIGTI0DQAQAaGmmQMRpoDIyDIaMJoBoDIDIMgMjI0aA0AMgaDQNAGQAAwQMmRoGgAgA0NNMgYjTQGRkGQ0YTQDQGQGQZAZGRo0BoAZA0GgaAMgABggZMjQNABABoaaZAxGmgMjIMhowmgGgMgMgyAyMjRoDQAyBoNA0AZAADBAyZGgaAAmqU1NEgJqnptU/Sn4jRR6J6epk2pqb1Q/SgAPUGgyNNGjQ2SBpoAZAAGg0NB6mgDIAAAAA2oaApSREBNAARhGiYEaEwU8pvImlP0k2aam1GaGqbFNM1MHpTwmkepmyU9R6nqPKekHqNNPUxNGhp6n6p6QaZ6o9TG1GMqcoV9ly6nRanHlq6zPNbnGZNi6HSug+2nPiZ13XcnFYZW+45W11CumhzYhchOJ2GLLV1OBjBjGf4TptOddTSOcVxhqYZMYwZXZZY00zI1paX5X9J+b+f4e+x43RXSxXPOdquiGpduatGyXneN696M9t4HU2eR5XX/kPhP261NTx3JO1Ow7LyuDmeo9a7d351T1ZxnvnrvYnrXv/hXxPCeuYx2XsNmO003eg9J3Z6U7b23meJ4ri01OdzTk9BNO96brz+qT5nuvvH3ds/G+m/JcG/F2XYuhXlvO+jP7U3XgrzPN/lr8Sf1n6j4j7jZs+s/T0tNaNNYzTs12rxjwztHlnire3Nzc3N1wuBwOBwXBvZfoHpD7rFmR99V5vj3aXza3xdBbXMalubTg/jIv5dfAi54Pdc75j4z412n3Npj3Ld/ENm7a3b/Cod6h/ret1/5vn/C+l+gdslMvgPSLJ8d8q+U66fevYn/tW1chleEtNTGlcHCbLRlq0tHzF5tsbbZZfHjjLgZu42XCuC3NrdjTasZGNzgxPIrGqp7r3p7L2p5XjnpPSmTd5XtzqnB6U87zzg1Ol0zd0zsLszxR6lkxp35u6/teL0L0W922cR7Lu1lpL9CsHirzuM2T+BgsyViT6LHcm0/Vr6U/7LGGyJeqTEjt0PHWhF5mCT7R9mtlDwriYv0Tyr/OxYt6qp5r0mPVT0608TqnqMZaarU2nFwrTzzlrs1ed7z1ux60wyr4ydCaTi3enW8x68x0zU7tXSlcmPSW1mGpWJMg4zmPC2lK96tp0OE80y4MfEvnZj8zGluR6b22ki1Ou9V2nCd9xovcPvcYMZYy0lvN60ScZ45vN6yeCeeXFb1lVjnnCar5fwXwE2bzJ4HI1XVPXfXZMm44GUsMpYsmLB65TuVdm0cl0b+i/wGNN66XjeV7zuPpHcnK/juhhjdfId5jMdE5nN0dGmmm2zZs2cexD5n9p/dY352XsvXHaZNWWsmmS1atjR452nYudzvqv2HMRyvNNnlMcDl3R2+yx2uVrBubTW9icHDVtbNXlZm7jma1rM4VurZZd2y6nUau7ZXZ7bVU+mnoOVxZGMrVmvX60605JwmzGZhhhjTWtaaaMaaGTGmNMZasY0iX8VMUl8eepaIrzGSpemWOQyZORk2bNpjUybMmxqYmknCGCFynutfksaZpjTNMaaatM0xsxcGR0sociNqxNSmhhR1ZJPbsn8qyF0t2qH6iYBclclalbtTTcHTDsPaX6rlnElph2Jyumumtynv2Kk8GI7rsvXbIcJgHJOSaSXnnGaI3m87RtVXJOZ/YtgdTE6Wpha6ZlE8ayXkef1fh602r2WwvfMXtMdLlkfnLFdYYwYso+bWqm7yJqHXZGw2nrS5ZanSYnWlxBxMF1V940K2wdrI7R6OYf7DGGamMmTSbRhlS45xmVOumF1EyPCmHrrN8wwZOOrdNtLeMtzFzDlWnfTBxMk2NaXIZHBYxYLD4w8yju0ao65Vz1OIXoS9dLanwCe1PWrYuWMqf1if1z2k2yYfKJ741PDgno1ZQ8DRqvUny3mNoWTzGO6m1DkrJI8JiR5cSd+vZdGOO8nrMoc5+NDUFsMSXaZJeNlMmGLtJsovOsUp7I9S5VojKxF6bTVEelXqlfJobQr3LozSh2Jk7VcrVMfhXqszGWMzNqGhqZY0OadxkyyMssKugZR0KNFXBHlqwmJgTE/BNVMk6ItJXZMR0H47GpXv/DMOvNkmVuaV1PRfEdxuqc7Hcd+ZV/zTLaRxWk0nl9CdCeM6mn5rstHIBcpiuwmUZXeq81DacHI2rmrZ5SuE5mOZd6LQrZg9mx32TprA8BMo5jKN6yLTCi3WzQaZSuhzTtM1fUTGVpG8Tw+KXI0tjEpiWxtLYynOlktSbVlaI5kxP8TDH8kx50xoxi5KcA4pcja8KWLRlO/Ks6q06ergnvm1ca3Tq8Uw7LTUsmWyctXPWmpitl/uvGcWTGXGuAXDfhqazGmjkxcJW5hMMMMpYsXl2TZYtVOddG3XCarUt6Ptq9CZXSNzyuRzqRZOjsxdBbFVz6OA5HI43r1jityVlVpVkxmOsyaYWE1NTGq1sOVh36mHMcxtSvcy70edG0ZGR3I1Go1GRlV7mWWo1G0ZGRqlvH40l7o4m5xMWLLLYyNjnqc8556mdPqLJ31n/1nWOncxzG1tizrHs/Z+d2vP/B/l8wdJ6rHUn2nbbDq4p6htFtYzMMMTaZis1K5GKzGNmxhmUx2DDlZ/qNnIx41xnaMfCZWYaZWtNLTNW8ND4Fw1MyZOCdM428suKG1ehW8TesOydg7J+YYcD4cYR+8dFK6M4E3HM9ZfRNNL+Sn6rsl4DsrDl2HpPCnfxjGXtbZtYys1ttlyJ4T+BvexjGWRjMszK4Jpc77D3GyuVD7q0+G8m9G+2+rGm7cOR2y7FdtY2XUYx/oNlfRYxhMYyYZkyyg55enna9Kt/FFi6GMMwYwdwxWgxGMLKYmUyGExTKMZkMFhkymKuh0NOBNnBu+23LdwDoZYYzGGMxtORaTU1pjTGWTTGGtMrNWUsyyTTLLG1qy2ZjbK2DBllWqxMtBMaYZQmcE7zvvRcTkclUwdkxTaSdyySt/7fpL+T1v516Ji97fwr5JbLu305zMn5+GMTTZ9F+y7ExwmGVfG44yxn3dLv6l5i+Wth1jCrDq21nW9LqvvDzz3Vf3LLH/O/32TJ/erx3bXftO4eF+G956D952K/An4NfvOpjFjExjevP/UmE0fIoZXx6/w6lX/no3D0bLt+ixjieBM6ksRd0yB4Lt2SwYNE+gd1detlZWUnpiZfGfFaK+4PyCa/v18V8X75pe9fLXzp7l3VjF76vWZmHwGz1IZNWT7b8yddJ4q5kyrVdfru6atWc7bVYztL9Jf4GXvT+Y8m9/YsXP6H018a8D4XVOqvfzqeR+6yZOD8dPv0+U7/q5Pl+2dNb0MjzGVH5p6MNQ7cOWvw62U9aHE8DprDek+McLyvDz+te+9Zhq5+YTruufMcWMabqysTmZVWjKPfnK0wyVcrsuhjZRdLkHNvD72b9abriOSGIxiLixMOoalNPXzy+wT/tf+U6HHONfsz+xe8ufHBdQWWGWLA9if0rsnmrxK5LvRZQeWsTCsrmOYy8VteVfuRfcVTtDLItLIsMYxZLdU/DbtSemxF6Z6Zo5WBXE4tFdCyVMMXMTEMZXVlS6Xec2T4e0tHsRcEuWshcJ2YsNF5rUx1E8ifCq6Z+ZP7qdCeu/aTwFd53l16/o0NOw6O3dLavP4Hbi4RdmuDk6DoYaninC0+o4uZjbJ7Rxeu0/FbuFg+q7DVS6fQe0rZ6NDGUNNU6DEqOaLTicKnYZMnBWruljQxoaS3dZhocDge0bSTyOvdAbG5hxe2xji7E/L55xX13wWNDi6HCekcFxfCPGxY0MXC+s7afWaMdDyjyr+o8Rudm/NabOZvdl274zH4f5XK9z6On1Pe/K5TdPAslg77BjuO6Y3eO7GqvOPG/stknp1leyvLL0Z7bl9I4noMvLkzytLhWYzrOZzLXCORe028rORzOg4N/L0HlMOQ3Pgmnbb6KczlabORpu980q37TBqRu0/p3PO6234Bl03Ynuz+9W7gnsEcmvYaYY3aMYY0wx3pYd+ujsXauWdaY5Xkbtl23fPzFHiDB/QMo0yFjBllYxTQYYyxkrwn7JufwJ/PfgJ+C83X69ni6zvXcnyXabv0ncbLwsceS+RNlyN2mnneJtX0ngYO0+e+0+UnA+Wch3ji8hj5an4h+i6XBySU4n+R0roVcbw5yvHrmr4Yw8Y7x6c+9POPYHI5HI5HI5HI5HGXGww4nE4nrVyOR8XeqPEO7PLOiukYa3Novk5hV4cdtYZLI93e+uxff2jRo0aNGjRo0aNG1bVtW1dy3m83m8+tQ5ZzHw3nObwOu8La9Rc1dtkdS8A3eTk823tnktXWlxN6Oixe06zrN70Isd9jiOgZFq9yfkPqP/SLhN2Myl8jDM43bl1nbcb4cO57jlh8Jow6pzXZdL4dyODTuuhu77FyO27DdwdRxmvO+O+3N2+BdqyTwLHVczDVY4UPE4O66/ZO2cx1LFzVdSXtF7G4HMbrauOHRw6c8FdZ5m9fHZHYZXfTlZquyynSyTTKke6vcffSD9pzPA/G7n7jxPmuhc1DHMynPMrGL6AdewYmwu5ko+UUyTwrMv27rPH1v1nGqd87+p6N6LU8k3NEng53xXyHS97+44OSg/sy/hn+Se6yfYNjW0/uTgP+PvWYzLMmjhcLB/gGpri6H83/84eUXWT6T9Hsv7785z/7z4icpW+zfXypuR7rx/gMdZb1/wC678pcs8/2a3mDitGHxl9mfPlll5MafWWqxk/eYuTDgcNMzDGWLWvsuglNxs53GtN6uWpktlW1tZZYcuinMMWmnNnJydze3b2Y1McBxrBkXw799izLMZZYyy0TkbsGM4p03S2uVu5s/XXUdSdec6smVxZYYGpVmT8A+8ajuEyV5FatkvVru2x6uxGXXbH4A+jvgP4GMYy3iPLXzq/6z65+E005ey+cwMZD3fZcqc6xpjTFjQ0P3U+e++cPYmTIwj0nrK5NPTfl3WvpfLtXDcb2HQMudYOxFXQBor4L4T6vrOauFctYXJQ++NUWmJe5bmx1jDiZS1dTqWxo4GR8jm3fttpmPHppk9PEyv4/y8/sO07XacOmcqc0x2Vi9BvNJvN5oW8x4mOsydpidRxMYJPx06m1bqPzq9KtK8sxXNXFodD/+MYYaJTLwOhc9brCsV18oOR1i4tXChyTkq4lf4y1Ke+9axjDHqs1mfBbMXuP4Hzi+X7t8vzv7bHerrUPgPCxhjre4fXdfLNtNM+Jd+Zdh8xd8wP87uNPoPgv4W7/5P2BuxfsMabNnMnza+54Pdi5U671GPZY8CehX8Voeoo7FHpkeEc6715FwHZrIrUrHaviPUbPZHND+IhczrP6FcYvhOZ0Di/ETt0OI+YwNWR9r7tpf6WDeZKZDB1+z2IthOl1mPyb5FluvEx9h9d0NnM0Y1XPFkWIsk1WotJ0PBMmkvjvQTd0e71tfeV+8r8lQ/tpzpsmxJ+InrI/dj2UajUajVTUajatRqNRtGo1Go1Go4wjeMpZFMVV9CHbofPraLsJ3JpWV2XOoanCuFky4y3PPNxucK2uKC1Lbdb1eo+m5XomN6HfeZsabHLHRX/K+offtNGGmHWctcVcG44MdSqsOLY9VzX+Zxfxn2HPdWTpzWvkrtJ8M5zorrKcquRytJ5N5DZmcaW02l76nWO+BqPXm1A2Ry/0q71dH/mqrqeFjkYxjEXtsX8qubTk67rGycyqsdm4tZx5D6D5hhi0waaWmiaMP81Yjii5qxPlPuU/GfTL1Y5E6Jyfiq63qTa39A4J0sOGDgO9WF9bOXl0XfPRbsY2bPNKPy1YrFYrFYmRhhlTIyMjJWJYZHXuCXI8OoXsvfljGLFicNifpp2XunoPiG1wtx3p1Tah+/DD66OnVtVXP9rKbVxOnL0tR/rHtqB5UDErUVcl11D4qqvjpOcxX7armUNJB3LpW6bxVvD08e8h3odKKvyCFZBdSh2FVcST9xV3n3T8t1j7Kr9qgrqXg+13Pt5U7JCvFXVIV1YG5lRhkVYZJYYDDD4KOIMoHCp26WS8GB7uBh2zIdgq/PKyInjV2STShuoapUdCpX1yTwqq/z1VvET7Kh5nVPkO8YyxjLt2MaaMmWTLQvx3qnzltnXW0p2jxgbEtSny/Osv8Y9pLMXYoHVPAhkVdWVeODhR6q9/Sxe2liwwZWMVvFXfRkeIDxAePUPIrdJ4ey6yquzH+PD/bUOWAu05qVHtFd8rrKHSoeNIOUqrYr3FXyToqfYJgwmJdKpXXOwYYegNNGMzfZPp/t3t/DVs4zjNTN61rRqaWaa4NYbRjTa0tWwy2Y2tGN8ZO8ofNKq4j9SL7I+cSm4/6ovLV5HNXLI0jJidwrtk6ynCaP6Z++GjRlWS3tLeW129Mi9evxU9mtz6s5J3Z7M2ngTgnKvmpomxpaLCzPfmx0JWE+m3NLDDGOX47RctdYYNK5jakdqLkRlI39n590T5zctGSwwZZDJj6kW8XSi6ot2MmWWJ0DUT3nuvebBudScjZ79g8cWJ8av0k+/bE5WKd5MdbFpbDVMxu1DVMmtNZGJvq1mtRbn6M+g/kP0FwDwr7quZs7xosNGpbscyxhhd9TyJyFwbLcxlTasg75vW7TsV5K7ji44XPMMrdoj+Y3rT0Hie62nlYV/pwczzOmdLqLhYkzGMzCZWGMQzGMSsZYY6Di1t4nlJ+Em63mJxrVLxPbYxNEdgc1dU2iOKyoYYWjNrEeHTYybVk0atSa7ehuwsWMWTqn1TrnS6hYsi71d1+s+k+ic70e20fzE/VaTdxT9ZtU4GIXdeNx3X77guYYfpHeTQjaMX6brOu4OY4K7Y2d9mbHarI5ox3p4GpJ2Vd/Tst60f7j999pppjR+Q/Qf8J/VaORs3cji7FfFuN61+ui9s8hix1OCh5KGVV23BPXvZfz3CLyHpix+exi8z/KnCnosY2eunor+cxyPO/xJ0vKey9OvE9VjqaYu0x3Z3jd6o2b1T12D+F8l232lwaaacD5LE8LBxu7WTlbWraWpew8Xexjel3E+wWD4APITdNqR8F3R3T0lunCQ4GaE9R37DxeCYfcHi4xci5ovKfxVs55y2hf+65E/Xdp6jR5nrebTmi5incpkyOjs50JvrZwstbbW6kfuuQw+2mykf/EXNFzxfKTrxew929TR6bWnGL//F3JFOFCQT3K4lQ"
+
+    kernels = Kernel(
+        bz2.decompress(base64.b64decode(quantization_code)),
+        [
+            "int4WeightCompression",
+            "int4WeightExtractionFloat",
+            "int4WeightExtractionHalf",
+            "int8WeightExtractionFloat",
+            "int8WeightExtractionHalf",
+        ],
+    )
+except Exception as exception:
+    kernels = None
+    logger.warning("Failed to load cpm_kernels:" + str(exception))
+
+
+class W8A16Linear(torch.autograd.Function):
+    @staticmethod
+    def forward(ctx, inp: torch.Tensor, quant_w: torch.Tensor, scale_w: torch.Tensor, weight_bit_width):
+        ctx.inp_shape = inp.size()
+        ctx.weight_bit_width = weight_bit_width
+        out_features = quant_w.size(0)
+        inp = inp.contiguous().view(-1, inp.size(-1))
+        weight = extract_weight_to_half(quant_w, scale_w, weight_bit_width)
+        ctx.weight_shape = weight.size()
+        output = inp.mm(weight.t())
+        ctx.save_for_backward(inp, quant_w, scale_w)
+        return output.view(*(ctx.inp_shape[:-1] + (out_features,)))
+
+    @staticmethod
+    def backward(ctx, grad_output: torch.Tensor):
+        inp, quant_w, scale_w = ctx.saved_tensors
+        weight = extract_weight_to_half(quant_w, scale_w, ctx.weight_bit_width)
+        grad_output = grad_output.contiguous().view(-1, weight.size(0))
+        grad_input = grad_output.mm(weight)
+        grad_weight = grad_output.t().mm(inp)
+        return grad_input.view(ctx.inp_shape), grad_weight.view(ctx.weight_shape), None, None
+
+
+def compress_int4_weight(weight: torch.Tensor):  # (n, m)
+    with torch.cuda.device(weight.device):
+        n, m = weight.size(0), weight.size(1)
+        assert m % 2 == 0
+        m = m // 2
+        out = torch.empty(n, m, dtype=torch.int8, device="cuda")
+        stream = torch.cuda.current_stream()
+
+        gridDim = (n, 1, 1)
+        blockDim = (min(round_up(m, 32), 1024), 1, 1)
+
+        kernels.int4WeightCompression(
+            gridDim,
+            blockDim,
+            0,
+            stream,
+            [ctypes.c_void_p(weight.data_ptr()), ctypes.c_void_p(out.data_ptr()), ctypes.c_int32(n), ctypes.c_int32(m)],
+        )
+        return out
+
+
+def extract_weight_to_half(weight: torch.Tensor, scale_list: torch.Tensor, source_bit_width: int):
+    if source_bit_width == 8:
+        func = kernels.int8WeightExtractionHalf
+    elif source_bit_width == 4:
+        func = kernels.int4WeightExtractionHalf
+    else:
+        assert False, "Unsupported bit-width"
+
+    with torch.cuda.device(weight.device):
+        n, m = weight.size(0), weight.size(1)
+        out = torch.empty(n, m * (8 // source_bit_width), dtype=torch.half, device="cuda")
+        stream = torch.cuda.current_stream()
+
+        gridDim = (n, 1, 1)
+        blockDim = (min(round_up(m, 32), 1024), 1, 1)
+
+        func(
+            gridDim,
+            blockDim,
+            0,
+            stream,
+            [
+                ctypes.c_void_p(weight.data_ptr()),
+                ctypes.c_void_p(scale_list.data_ptr()),
+                ctypes.c_void_p(out.data_ptr()),
+                ctypes.c_int32(n),
+                ctypes.c_int32(m),
+            ],
+        )
+        return out
+
+
+class QuantizedLinear(Linear):
+    def __init__(self, weight_bit_width: int, weight_tensor=None, bias_tensor=None, empty_init=False, *args, **kwargs):
+        super(QuantizedLinear, self).__init__(*args, **kwargs)
+        self.weight_bit_width = weight_bit_width
+
+        shape = self.weight.shape
+        del self.weight
+
+        if weight_tensor is None or empty_init:
+            self.weight = torch.empty(
+                shape[0], shape[1] * weight_bit_width // 8, dtype=torch.int8, device=kwargs["device"]
+            )
+            self.weight_scale = torch.empty(shape[0], dtype=kwargs["dtype"], device=kwargs["device"])
+        else:
+            self.weight_scale = (weight_tensor.abs().max(dim=-1).values / ((2 ** (weight_bit_width - 1)) - 1)).half()
+            self.weight = torch.round(weight_tensor / self.weight_scale[:, None]).to(torch.int8)
+            if weight_bit_width == 4:
+                self.weight = compress_int4_weight(self.weight)
+
+        self.weight = Parameter(self.weight.to(kwargs["device"]), requires_grad=False)
+        self.weight_scale = Parameter(self.weight_scale.to(kwargs["device"]), requires_grad=False)
+        if bias_tensor is not None:
+            self.bias = Parameter(bias_tensor.to(kwargs["device"]), requires_grad=False)
+        else:
+            self.bias = None
+
+    def forward(self, input):
+        output = W8A16Linear.apply(input, self.weight, self.weight_scale, self.weight_bit_width)
+        if self.bias is not None:
+            output = output + self.bias
+        return output
+
+
+def quantize(model, weight_bit_width, empty_init=False, **kwargs):
+    """Replace fp16 linear with quantized linear"""
+
+    for layer in model.layers:
+        layer.attention.query_key_value = QuantizedLinear(
+            weight_bit_width=weight_bit_width,
+            weight_tensor=layer.attention.query_key_value.weight.to(torch.cuda.current_device()),
+            bias_tensor=layer.attention.query_key_value.bias,
+            in_features=layer.attention.query_key_value.in_features,
+            out_features=layer.attention.query_key_value.out_features,
+            bias=True,
+            dtype=torch.half,
+            device=layer.attention.query_key_value.weight.device,
+            empty_init=empty_init
+        )
+        layer.attention.dense = QuantizedLinear(
+            weight_bit_width=weight_bit_width,
+            weight_tensor=layer.attention.dense.weight.to(torch.cuda.current_device()),
+            bias_tensor=layer.attention.dense.bias,
+            in_features=layer.attention.dense.in_features,
+            out_features=layer.attention.dense.out_features,
+            bias=True,
+            dtype=torch.half,
+            device=layer.attention.dense.weight.device,
+            empty_init=empty_init
+        )
+        layer.mlp.dense_h_to_4h = QuantizedLinear(
+            weight_bit_width=weight_bit_width,
+            weight_tensor=layer.mlp.dense_h_to_4h.weight.to(torch.cuda.current_device()),
+            bias_tensor=layer.mlp.dense_h_to_4h.bias,
+            in_features=layer.mlp.dense_h_to_4h.in_features,
+            out_features=layer.mlp.dense_h_to_4h.out_features,
+            bias=True,
+            dtype=torch.half,
+            device=layer.mlp.dense_h_to_4h.weight.device,
+            empty_init=empty_init
+        )
+        layer.mlp.dense_4h_to_h = QuantizedLinear(
+            weight_bit_width=weight_bit_width,
+            weight_tensor=layer.mlp.dense_4h_to_h.weight.to(torch.cuda.current_device()),
+            bias_tensor=layer.mlp.dense_4h_to_h.bias,
+            in_features=layer.mlp.dense_4h_to_h.in_features,
+            out_features=layer.mlp.dense_4h_to_h.out_features,
+            bias=True,
+            dtype=torch.half,
+            device=layer.mlp.dense_4h_to_h.weight.device,
+            empty_init=empty_init
+        )
+    return model

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "bos_token": "<sop>",
+  "eos_token": "<eop>",
+  "mask_token": "[MASK]",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

test_modeling_chatglm.py

@@ -0,0 +1,165 @@
+import datetime
+import math
+import unittest
+import torch
+import random
+
+from transformers import AutoTokenizer, AutoModel
+from transformers.testing_utils import require_torch, slow, torch_device
+
+
+def set_random_seed(seed):
+    import random
+
+    random.seed(seed)
+
+    # pytorch RNGs
+    import torch
+
+    torch.manual_seed(seed)
+    torch.backends.cudnn.deterministic = True
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed_all(seed)
+
+    # numpy RNG
+    import numpy as np
+
+    np.random.seed(seed)
+
+
+
+def ids_tensor(shape, vocab_size):
+    #  Creates a random int32 tensor of the shape within the vocab size
+    total_dims = 1
+    for dim in shape:
+        total_dims *= dim
+
+    values = []
+    for _ in range(total_dims):
+        values.append(random.randint(0, vocab_size - 1))
+
+    return torch.tensor(data=values, dtype=torch.long, device=torch_device).view(shape).contiguous()
+
+
+def get_model_and_tokenizer():
+    model = AutoModel.from_pretrained("/mnt/vepfs/workspace/zxdu/chatglm_6b", trust_remote_code=True).half()
+    model.to(torch_device)
+    model.eval()
+    tokenizer = AutoTokenizer.from_pretrained("/mnt/vepfs/workspace/zxdu/chatglm_6b", trust_remote_code=True)
+    return model, tokenizer
+
+
+@require_torch
+class ChatGLMGenerationTest(unittest.TestCase):
+    def get_generation_kwargs(self):
+        pass
+
+    def test_chat(self):
+        model, tokenizer = get_model_and_tokenizer()
+        prompts = ["你好", "介绍一下清华大学", "它创建于哪一年"]
+        history = []
+        set_random_seed(42)
+        expected_responses = [
+            '你好👋！我是人工智能助手 ChatGLM-6B，很高兴见到你，欢迎问我任何问题。',
+            '清华大学是中国著名的综合性研究型大学，位于中国北京市海淀区，创建于 1911 年，前身是清华学堂。作为我国顶尖高等教育机构之一，清华大学在科学研究、工程技术、信息技术、经济管理等领域处于领先地位，也是世界上最著名的工程学府之一。\n\n清华大学拥有世界一流的教学设施和科学研究平台，设有多个学院和研究中心，包括工程学院、自然科学学院、社会科学学院、人文学院、法学院、经济管理学院等。学校拥有众多知名教授和研究团队，其中包括多位院士、国家杰出青年科学基金获得者、长江学者等。\n\n清华大学的本科生招生范围为全国中学毕业生，本科生入学要求严格，考试成绩优秀。同时，清华大学也提供研究生和博士生招生，包括硕士研究生和博士研究生。',
+            '清华大学创建于 1911 年。'
+        ]
+        for (prompt, expected_response) in zip(prompts, expected_responses):
+            response, history = model.chat(tokenizer, prompt, history=history)
+            print(repr(response))
+            self.assertEquals(expected_response, response)
+
+    def test_stream_chat(self):
+        model, tokenizer = get_model_and_tokenizer()
+        prompts = ["你好", "介绍一下清华大学", "它创建于哪一年"]
+        history = []
+        expected_responses = [
+            '你好👋！我是人工智能助手 ChatGLM-6B，很高兴见到你，欢迎问我任何问题。',
+            '清华大学是中国著名的综合性研究型大学，位于中国北京市海淀区，创建于 1911 年，前身是清华学堂。作为我国顶尖高等教育机构之一，清华大学在科学研究、工程技术、信息技术、经济管理等领域处于领先地位，也是世界上最著名的工程学府之一。\n\n清华大学拥有世界一流的教学设施和科学研究平台，设有多个学院和研究中心，包括工程学院、自然科学学院、社会科学学院、人文学院、法学院、经济管理学院等。学校拥有众多知名教授和研究团队，其中包括多位院士、国家杰出青年科学基金获得者、长江学者等。\n\n清华大学的本科生招生范围为全国中学毕业生，本科生入学要求严格，考试成绩优秀。同时，清华大学也提供研究生和博士生招生，包括硕士研究生和博士研究生。',
+            '清华大学创建于 1911 年。'
+        ]
+        set_random_seed(42)
+        for prompt, expected_response in zip(prompts, expected_responses):
+            response = ""
+            for idx, (response, history) in enumerate(model.stream_chat(tokenizer, prompt, history=history)):
+                pass
+            print(repr(response))
+            self.assertEquals(expected_response, response)
+
+    def test_generation(self):
+        model, tokenizer = get_model_and_tokenizer()
+        sentence = "晚上睡不着怎么办"
+        parameters = [(False, 2048, 1),
+                      (False, 64, 1),
+                      (True, 2048, 1),
+                      (True, 64, 1),
+                      (True, 2048, 4)]
+        expected_out_sentences = [
+            '晚上睡不着怎么办 以下��一些可能有助于在晚上入睡的方法:\n\n1. 保持规律的睡眠时间表:尽量在同一时间上床,并尝试在早上醒来时自然起床。\n\n2. 创建舒适的睡眠环境:保持房间安静、凉爽、黑暗、舒适,并使用舒适的床垫和枕头。\n\n3. 避免刺激性物质:避免饮用含咖啡因的饮料,如咖啡、茶和可乐,并尽可能减少饮酒。\n\n4. 放松身心:尝试进行放松的活动,如冥想、深呼吸、瑜伽或听轻柔的音乐。\n\n5. 避免在床上做其他事情:例如看电视、使用电脑或智能手机等。\n\n6. 练习放松技巧:例如渐进性肌肉松弛法、冥想或深呼吸练习。\n\n7. 寻求帮助:如果长时间都无法正常入睡,可以考虑咨询医生或专业心理医生,寻求更进一步的帮助。\n\n希望这些方法能有助于入睡。',
+            '晚上睡不着怎么办 以下是一些可能有助于在晚上入睡的方法:\n\n1. 保持规律的睡眠时间表:尽量在同一时间上床,并尝试在早上醒来时自然起床。\n\n2. 创建舒适的睡眠环境:保持房间安静、凉爽、黑暗、舒适,并使用舒适的床垫和枕头。',
+            '晚上睡不着怎么办 以下是一些有助于在晚上更好地入睡的方法:\n\n1. 维持规律的睡眠时间:每晚尽可能在同一时间上床,保持规律的睡眠时间表,帮助身体调整并更容易入睡。\n\n2. 避免在床上使用电子设备:手机、平板电脑、电脑等电子设备会发出蓝光,这会干扰身体释放褪黑素,进而导致难以入睡。建议你在睡前一小时停止使用这些设备。\n\n3. 创建舒适的睡眠环境:确保卧室安静、黑暗、凉爽,舒适的床垫和枕头,保持卧室温度适宜,这有助于让你更容易入睡。\n\n4. 放松身心:尝试进行一些放松的活动,如冥想、深呼吸、瑜伽或轻松的散步,减轻压力和焦虑,让你更容易入睡。\n\n5. 避免咖啡因和酒精:咖啡因和酒精会让大脑更加兴奋,进而干扰身体入睡过程。建议在睡前几小时避免饮用这些物质。\n\n6. 做一些安静的活动:阅读一本书、听轻柔的音乐、绣或者绘画等安静的活动,有助于自己放松身心,进而更容易入睡。\n\n如果采取以上这些方法仍然无法入睡,建议咨询医生或专业的睡眠专家,获取更好的建议和帮助。',
+            '晚上睡不着怎么办 以下是一些有助于在晚上更好地入睡的方法:\n\n1. 维持规律的睡眠时间:每晚尽可能在同一时间上床,保持规律的睡眠时间表,帮助身体调整并更容易入睡。\n\n2. 避免在床上使用电子设备:手机、平板电脑、电脑等电子设备会发出蓝光,这会干扰身体',
+            '晚上睡不着怎么办 以下是一些可能有助于在晚上入睡的方法:\n\n1. 建立规律的睡眠时间表:尽量在同一时间入睡和起床,即使在周末和假期也要尽量保持一致。\n\n2. 创造舒适的睡眠环境:保持房间安静、凉爽、黑暗、舒适,使用舒适的床垫和枕头等。\n\n3. 放松身心:尝试进行一些放松的活动,如冥想、深呼吸、瑜伽、听轻柔的音乐等,缓解压力和紧张情绪。\n\n4. 避免刺激性物质:避免饮用咖啡、茶、可乐等含咖啡因的饮料,避免吸烟和饮酒等刺激性物质。\n\n5. 避免躺在床上翻来覆去:如果躺在床上超过20分钟还不能入睡,就不要躺在床上翻来覆去,而是起床去做一些放松的活动,直到感到困倦为止。\n\n6. 练习放松技巧:如果感到焦虑或紧张,可以尝试进行一些放松技巧,如渐进性肌肉松弛、冥想等。\n\n7. 改善睡眠障碍:如果已经尝试了上述方法仍然无法入睡,可以考虑咨询医生,了解是否存在其他睡眠障碍问题,并接受相应的治疗。']
+        for (do_sample, max_length, num_beams), expected_output_sentence in zip(parameters, expected_out_sentences):
+            set_random_seed(42)
+            inputs = tokenizer(sentence, return_tensors="pt")
+            inputs = inputs.to(torch_device)
+
+            outputs = model.generate(
+                **inputs,
+                do_sample=do_sample,
+                max_length=max_length,
+                num_beams=num_beams
+            )
+
+            outputs = outputs.tolist()[0]
+            out_sentence = tokenizer.decode(outputs, skip_special_tokens=True)
+            print(out_sentence)
+            self.assertEquals(expected_output_sentence, out_sentence)
+
+    def test_batch_generation(self):
+        model, tokenizer = get_model_and_tokenizer()
+        sentences = [
+            "你好",
+            "介绍一下清华大学"
+        ]
+        parameters = [(False, 2048, 1),
+                      (False, 64, 1),
+                      (True, 2048, 1),
+                      (True, 64, 1),
+                      (True, 2048, 4)]
+        expected_out_sentences = [
+            ['你好 你好👋!我是人工智能助手 ChatGLM-6B,很高兴见到你,欢迎问我任何问题。',
+             '介绍一下清华大学 清华大学��中国著名的综合性大学,位于北京市海淀区双清路30号,其历史可以追溯到1911年创建的清华学堂,1925年更名为清华学校,1937年抗日战争全面爆发后南迁长沙,1946年迁回清华园。新中国成立后,清华学校更名为清华大学。\n\n清华大学是中国最顶尖的大学之一,在工程、科学、技术、经济、管理等领域都有很高的学术声誉和影响力。学校拥有世界一流的教学设施和科学研究平台,有多个学院和研究中心,包括工程学院、自然科学学院、人文学院、社会科学学院、经济管理学院、法学院、美术学院、医学院、器学院等。\n\n清华大学的本科生招生始于2000年,实行全面二孩政策后,本科生招生规模不断扩大。截至2022年,清华大学共有本科生近3万人,研究生近2万人,其中国际学生占比约为10%。清华大学的本科生教育注重通识教育和个性化培养,强调实践、创新、国际化和综合素质。'],
+            [
+                '你好 你好👋!我是人工智能助手 ChatGLM-6B,很高兴见到你,欢迎问我任何问题。',
+                '介绍一下清华大学 清华大学是中国著名的综合性大学,位于北京市海淀区双清路30号,其历史可以追溯到1911年创建的清华学堂,1925年更名为清华学校,1937年抗日战争全面爆发后南迁长沙,1946年迁回'
+            ],
+            [
+                '你好 你好👋!我是人工智能助手 ChatGLM-6B,很高兴见到你,欢迎问我任何问题。',
+                '介绍一下清华大学 清华大学是中国著名的综合性研究型大学,位于北京市海淀区双清路 30 号,其溯源于 1911 年创建的清华学堂, 1925 年更名为清华学校, 1937 年秋抗日战争全面爆发后闭校。1949 年 10 月开学复校,成为我国第一个社会主义大学生活了的高校。截至 2023 年,清华学校共管辖 2 个学院、13 个系,有本科专业 60 个,研究生专业 190 个。'
+            ],
+            [
+                '你好 你好👋!我是人工智能助手 ChatGLM-6B,很高兴见到你,欢迎问我任何问题。',
+                '介绍一下清华大学 清华大学是中国著名的综合性研究型大学,位于北京市海淀区双清路 30 号,其溯源于 1911 年创建的清华学堂, 1925 年更名为清华学校, 1937 年秋抗日战争全面爆发后'
+            ],
+            [
+                '你好 你好👋!我是人工智能助手 ChatGLM-6B,很高兴见到你,欢迎问我任何问题。',
+                '介绍一下清华大学 清华大学是中国著名的综合性研究型大学,位于北京市海淀区双清路30号,其历史可以追溯到1911年创建的清华学堂,1925年更名为清华学校,1937年抗日战争全面爆发后南迁长沙,与北京大学、南开大学组建国立长沙临时大学,1938年迁至 昆明改名为国立西南联合大学,1946年迁回北京。新中国成立后,清华学校更名为清华大学。'
+            ]
+        ]
+        for (do_sample, max_length, num_beams), expected_output_sentence in zip(parameters, expected_out_sentences):
+            set_random_seed(42)
+            inputs = tokenizer(sentences, return_tensors="pt", padding=True)
+            inputs = inputs.to(torch_device)
+
+            outputs = model.generate(
+                **inputs,
+                do_sample=do_sample,
+                max_length=max_length,
+                num_beams=num_beams
+            )
+
+            batch_out_sentence = tokenizer.batch_decode(outputs, skip_special_tokens=True)
+            print(batch_out_sentence)
+            self.assertListEqual(expected_output_sentence, batch_out_sentence)

tokenization_chatglm.py

@@ -0,0 +1,443 @@
+"""Tokenization classes for ChatGLM."""
+from typing import List, Optional, Union
+import os
+
+from transformers.tokenization_utils import PreTrainedTokenizer
+from transformers.utils import logging, PaddingStrategy
+from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
+from typing import Dict
+import sentencepiece as spm
+import numpy as np
+
+logger = logging.get_logger(__name__)
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+    "THUDM/chatglm-6b": 2048,
+}
+
+
+class TextTokenizer:
+    def __init__(self, model_path):
+        self.sp = spm.SentencePieceProcessor()
+        self.sp.Load(model_path)
+        self.num_tokens = self.sp.vocab_size()
+
+    def encode(self, text):
+        return self.sp.EncodeAsIds(text)
+
+    def decode(self, ids: List[int]):
+        return self.sp.DecodeIds(ids)
+
+    def tokenize(self, text):
+        return self.sp.EncodeAsPieces(text)
+
+    def convert_tokens_to_string(self, tokens):
+        return self.sp.DecodePieces(tokens)
+
+    def convert_tokens_to_ids(self, tokens):
+        return [self.sp.PieceToId(token) for token in tokens]
+
+    def convert_token_to_id(self, token):
+        return self.sp.PieceToId(token)
+
+    def convert_id_to_token(self, idx):
+        return self.sp.IdToPiece(idx)
+
+    def __len__(self):
+        return self.num_tokens
+
+
+class SPTokenizer:
+    def __init__(
+            self,
+            vocab_file,
+            num_image_tokens=20000,
+            max_blank_length=80,
+            byte_fallback=True,
+    ):
+        assert vocab_file is not None
+        self.vocab_file = vocab_file
+        self.num_image_tokens = num_image_tokens
+        self.special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "<unused_0>", "<sop>", "<eop>", "<ENC>", "<dBLOCK>"]
+        self.max_blank_length = max_blank_length
+        self.byte_fallback = byte_fallback
+        self.text_tokenizer = TextTokenizer(vocab_file)
+
+    def _get_text_tokenizer(self):
+        return self.text_tokenizer
+
+    @staticmethod
+    def get_blank_token(length: int):
+        assert length >= 2
+        return f"<|blank_{length}|>"
+
+    @staticmethod
+    def get_tab_token():
+        return f"<|tab|>"
+
+    @property
+    def num_text_tokens(self):
+        return self.text_tokenizer.num_tokens
+
+    @property
+    def num_tokens(self):
+        return self.num_image_tokens + self.num_text_tokens
+
+    @staticmethod
+    def _encode_whitespaces(text: str, max_len: int = 80):
+        text = text.replace("\t", SPTokenizer.get_tab_token())
+        for i in range(max_len, 1, -1):
+            text = text.replace(" " * i, SPTokenizer.get_blank_token(i))
+        return text
+
+    def _preprocess(self, text: str, linebreak=True, whitespaces=True):
+        if linebreak:
+            text = text.replace("\n", "<n>")
+        if whitespaces:
+            text = self._encode_whitespaces(text, max_len=self.max_blank_length)
+        return text
+
+    def encode(
+            self, text: str, linebreak=True, whitespaces=True, add_dummy_prefix=True
+    ) -> List[int]:
+        """
+        @param text: Text to encode.
+        @param linebreak: Whether to encode newline (\n) in text.
+        @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding.
+        @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text.
+        @param add_dummy_prefix: Whether to add dummy blank space in the beginning.
+        """
+        text = self._preprocess(text, linebreak, whitespaces)
+        if not add_dummy_prefix:
+            text = "<n>" + text
+        tmp = self._get_text_tokenizer().encode(text)
+        tokens = [x + self.num_image_tokens for x in tmp]
+        return tokens if add_dummy_prefix else tokens[2:]
+
+    def postprocess(self, text):
+        text = text.replace("<n>", "\n")
+        text = text.replace(SPTokenizer.get_tab_token(), "\t")
+        for i in range(2, self.max_blank_length + 1):
+            text = text.replace(self.get_blank_token(i), " " * i)
+        return text
+
+    def decode(self, text_ids: List[int]) -> str:
+        ids = [int(_id) - self.num_image_tokens for _id in text_ids]
+        ids = [_id for _id in ids if _id >= 0]
+        text = self._get_text_tokenizer().decode(ids)
+        text = self.postprocess(text)
+        return text
+
+    def decode_tokens(self, tokens: List[str]) -> str:
+        text = self._get_text_tokenizer().convert_tokens_to_string(tokens)
+        text = self.postprocess(text)
+        return text
+
+    def tokenize(
+            self, text: str, linebreak=True, whitespaces=True, add_dummy_prefix=True
+    ) -> List[str]:
+        """
+        @param text: Text to encode.
+        @param linebreak: Whether to encode newline (\n) in text.
+        @param whitespaces: Whether to encode multiple whitespaces or tab in text, useful for source code encoding.
+        @param special_tokens: Whether to encode special token ([MASK], [gMASK], etc.) in text.
+        @param add_dummy_prefix: Whether to add dummy blank space in the beginning.
+        """
+        text = self._preprocess(text, linebreak, whitespaces)
+        if not add_dummy_prefix:
+            text = "<n>" + text
+        tokens = self._get_text_tokenizer().tokenize(text)
+        return tokens if add_dummy_prefix else tokens[2:]
+
+    def __getitem__(self, x: Union[int, str]):
+        if isinstance(x, int):
+            if x < self.num_image_tokens:
+                return "<image_{}>".format(x)
+            else:
+                return self.text_tokenizer.convert_id_to_token(x - self.num_image_tokens)
+        elif isinstance(x, str):
+            if x.startswith("<image_") and x.endswith(">") and x[7:-1].isdigit():
+                return int(x[7:-1])
+            else:
+                return self.text_tokenizer.convert_token_to_id(x) + self.num_image_tokens
+        else:
+            raise ValueError("The key should be str or int.")
+
+
+class ChatGLMTokenizer(PreTrainedTokenizer):
+    """
+    Construct a ChatGLM tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+    """
+
+    vocab_files_names = {"vocab_file": "ice_text.model"}
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    model_input_names = ["input_ids", "attention_mask", "position_ids"]
+
+    def __init__(
+            self,
+            vocab_file,
+            do_lower_case=False,
+            remove_space=False,
+            bos_token='<sop>',
+            eos_token='<eop>',
+            end_token='</s>',
+            mask_token='[MASK]',
+            gmask_token='[gMASK]',
+            padding_side="left",
+            pad_token="<pad>",
+            unk_token="<unk>",
+            num_image_tokens=20000,
+            **kwargs
+    ) -> None:
+        super().__init__(
+            do_lower_case=do_lower_case,
+            remove_space=remove_space,
+            padding_side=padding_side,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            end_token=end_token,
+            mask_token=mask_token,
+            gmask_token=gmask_token,
+            pad_token=pad_token,
+            unk_token=unk_token,
+            num_image_tokens=num_image_tokens,
+            **kwargs
+        )
+
+        self.do_lower_case = do_lower_case
+        self.remove_space = remove_space
+        self.vocab_file = vocab_file
+
+        self.bos_token = bos_token
+        self.eos_token = eos_token
+        self.end_token = end_token
+        self.mask_token = mask_token
+        self.gmask_token = gmask_token
+
+        self.sp_tokenizer = SPTokenizer(vocab_file, num_image_tokens=num_image_tokens)
+
+        """ Initialisation """
+
+    @property
+    def gmask_token_id(self) -> Optional[int]:
+        if self.gmask_token is None:
+            return None
+        return self.convert_tokens_to_ids(self.gmask_token)
+
+    @property
+    def end_token_id(self) -> Optional[int]:
+        """
+        `Optional[int]`: Id of the end of context token in the vocabulary. Returns `None` if the token has not been
+        set.
+        """
+        if self.end_token is None:
+            return None
+        return self.convert_tokens_to_ids(self.end_token)
+
+    @property
+    def vocab_size(self):
+        """ Returns vocab size """
+        return self.sp_tokenizer.num_tokens
+
+    def get_vocab(self):
+        """ Returns vocab as a dict """
+        vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def preprocess_text(self, inputs):
+        if self.remove_space:
+            outputs = " ".join(inputs.strip().split())
+        else:
+            outputs = inputs
+
+        if self.do_lower_case:
+            outputs = outputs.lower()
+
+        return outputs
+
+    def _tokenize(self, text, **kwargs):
+        """ Returns a tokenized string. """
+        text = self.preprocess_text(text)
+
+        seq = self.sp_tokenizer.tokenize(text)
+
+        return seq
+
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:
+        return self.sp_tokenizer.decode_tokens(tokens)
+
+    def _decode(
+            self,
+            token_ids: Union[int, List[int]],
+            **kwargs
+    ) -> str:
+        if isinstance(token_ids, int):
+            token_ids = [token_ids]
+        if len(token_ids) == 0:
+            return ""
+        if self.pad_token_id in token_ids:  # remove pad
+            token_ids = list(filter((self.pad_token_id).__ne__, token_ids))
+        return super()._decode(token_ids, **kwargs)
+
+    def _convert_token_to_id(self, token):
+        """ Converts a token (str) in an id using the vocab. """
+        return self.sp_tokenizer[token]
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        return self.sp_tokenizer[index]
+
+    def save_vocabulary(self, save_directory, filename_prefix=None):
+        """
+        Save the vocabulary and special tokens file to a directory.
+
+        Args:
+            save_directory (`str`):
+                The directory in which to save the vocabulary.
+            filename_prefix (`str`, *optional*):
+                An optional prefix to add to the named of the saved files.
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        if os.path.isdir(save_directory):
+            vocab_file = os.path.join(
+                save_directory, self.vocab_files_names["vocab_file"]
+            )
+        else:
+            vocab_file = save_directory
+
+        with open(self.vocab_file, 'rb') as fin:
+            proto_str = fin.read()
+
+        with open(vocab_file, "wb") as writer:
+            writer.write(proto_str)
+
+        return (vocab_file,)
+
+    def build_inputs_with_special_tokens(
+            self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
+        adding special tokens. A BERT sequence has the following format:
+
+        - single sequence: `[CLS] X [SEP]`
+        - pair of sequences: `[CLS] A [SEP] B [SEP]`
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs to which the special tokens will be added.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
+        """
+        gmask_id = self.sp_tokenizer[self.gmask_token]
+        eos_id = self.sp_tokenizer[self.eos_token]
+        token_ids_0 = token_ids_0 + [gmask_id, self.sp_tokenizer[self.bos_token]]
+        if token_ids_1 is not None:
+            token_ids_0 = token_ids_0 + token_ids_1 + [eos_id]
+        return token_ids_0
+
+    def _pad(
+            self,
+            encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
+            max_length: Optional[int] = None,
+            padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
+            pad_to_multiple_of: Optional[int] = None,
+            return_attention_mask: Optional[bool] = None,
+    ) -> dict:
+        """
+        Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
+
+        Args:
+            encoded_inputs:
+                Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
+            max_length: maximum length of the returned list and optionally padding length (see below).
+                Will truncate by taking into account the special tokens.
+            padding_strategy: PaddingStrategy to use for padding.
+
+                - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
+                - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
+                - PaddingStrategy.DO_NOT_PAD: Do not pad
+                The tokenizer padding sides are defined in self.padding_side:
+
+                    - 'left': pads on the left of the sequences
+                    - 'right': pads on the right of the sequences
+            pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
+                This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
+                `>= 7.5` (Volta).
+            return_attention_mask:
+                (optional) Set to False to avoid returning attention mask (default: set to model specifics)
+        """
+        # Load from model defaults
+        bos_token_id = self.sp_tokenizer[self.bos_token]
+        mask_token_id = self.sp_tokenizer[self.mask_token]
+        gmask_token_id = self.sp_tokenizer[self.gmask_token]
+        assert self.padding_side == "left"
+
+        required_input = encoded_inputs[self.model_input_names[0]]
+        seq_length = len(required_input)
+
+        if padding_strategy == PaddingStrategy.LONGEST:
+            max_length = len(required_input)
+
+        if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
+            max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
+
+        needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) != max_length
+
+        # Initialize attention mask if not present.
+        if max_length is not None:
+            if "attention_mask" not in encoded_inputs:
+                if bos_token_id in required_input:
+                    context_length = required_input.index(bos_token_id)
+                else:
+                    context_length = seq_length
+                attention_mask = np.ones((1, seq_length, seq_length))
+                attention_mask = np.tril(attention_mask)
+                attention_mask[:, :, :context_length] = 1
+                attention_mask = np.bool_(attention_mask < 0.5)
+                encoded_inputs["attention_mask"] = attention_mask
+
+            if "position_ids" not in encoded_inputs:
+                if bos_token_id in required_input:
+                    context_length = required_input.index(bos_token_id)
+                else:
+                    context_length = seq_length
+                position_ids = np.arange(seq_length, dtype=np.int64)
+                mask_token = mask_token_id if mask_token_id in required_input else gmask_token_id
+                if mask_token in required_input:
+                    mask_position = required_input.index(mask_token)
+                    position_ids[context_length:] = mask_position
+                block_position_ids = np.concatenate(
+                    [np.zeros(context_length, dtype=np.int64),
+                     np.arange(1, seq_length - context_length + 1, dtype=np.int64)])
+                encoded_inputs["position_ids"] = np.stack([position_ids, block_position_ids], axis=0)
+
+        if needs_to_be_padded:
+            difference = max_length - len(required_input)
+
+            if "attention_mask" in encoded_inputs:
+                encoded_inputs["attention_mask"] = np.pad(encoded_inputs["attention_mask"],
+                                                          pad_width=[(0, 0), (difference, 0), (difference, 0)],
+                                                          mode='constant', constant_values=True)
+            if "token_type_ids" in encoded_inputs:
+                encoded_inputs["token_type_ids"] = [self.pad_token_type_id] * difference + encoded_inputs[
+                    "token_type_ids"
+                ]
+            if "special_tokens_mask" in encoded_inputs:
+                encoded_inputs["special_tokens_mask"] = [1] * difference + encoded_inputs["special_tokens_mask"]
+            if "position_ids" in encoded_inputs:
+                encoded_inputs["position_ids"] = np.pad(encoded_inputs["position_ids"],
+                                                        pad_width=[(0, 0), (difference, 0)])
+            encoded_inputs[self.model_input_names[0]] = [self.pad_token_id] * difference + required_input
+
+        return encoded_inputs

tokenizer_config.json

@@ -0,0 +1,22 @@
+{
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_chatglm.ChatGLMTokenizer",
+      null
+    ]
+  },
+  "bos_token": "<sop>",
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": false,
+  "end_token": "</s>",
+  "eos_token": "<eop>",
+  "gmask_token": "[gMASK]",
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "num_image_tokens": 0,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "remove_space": false,
+  "tokenizer_class": "ChatGLMTokenizer",
+  "unk_token": "<unk>"
+}