zxdu20 commited on
Commit
e3f9625
1 Parent(s): 4874051

init commit

Browse files
README.md ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - zh
4
+ tags:
5
+ - glm
6
+ ---
7
+ GLM is a General Language Model pretrained with an autoregressive blank-filling objective and can be finetuned on various natural language understanding and generation tasks.
8
+
9
+ Please refer to our paper for a detailed description of GLM:
10
+
11
+ [GLM: General Language Model Pretraining with Autoregressive Blank Infilling](https://arxiv.org/abs/2103.10360) (ACL 2022)
12
+
13
+ Zhengxiao Du*, Yujie Qian*, Xiao Liu, Ming Ding, Jiezhong Qiu, Zhilin Yang, Jie Tang (*: equal contribution)
14
+
15
+ Find more examples in our [Github repo](https://github.com/THUDM/GLM).
16
+
17
+ ## Model description
18
+ `glm-large-chinese` is pretrained on the [WuDaoCorpora](https://www.sciencedirect.com/science/article/pii/S2666651021000152) dataset. It has 24 transformer layers, with hidden size 1024 and 16 attention heads in each layer. The model is pretrained with autoregressive blank filling objectives designed for natural language understanding, seq2seq, and language modeling.
19
+
20
+ ## How to use
21
+ Please refer the [instruction](https://github.com/THUDM/GLM#hugging-face-hub) in our Github repo.
22
+
23
+ We use three different mask tokens for different tasks: `[MASK]` for short blank filling, `[sMASK]` for sentence filling, and `[gMASK]` for left to right generation. You can find examples about different masks from [here](https://github.com/THUDM/GLM#left-to-right-generation--blank-filling-interactive). The prediction always begin with a special `<|startofpiece|>` token and ends with a `<|endofpiece|>` token.
24
+
25
+ ## Citation
26
+ Please cite our paper if you find this code useful for your research:
27
+ ```
28
+ @article{DBLP:conf/acl/DuQLDQY022,
29
+ author = {Zhengxiao Du and
30
+ Yujie Qian and
31
+ Xiao Liu and
32
+ Ming Ding and
33
+ Jiezhong Qiu and
34
+ Zhilin Yang and
35
+ Jie Tang},
36
+ title = {{GLM:} General Language Model Pretraining with Autoregressive Blank Infilling},
37
+ booktitle = {Proceedings of the 60th Annual Meeting of the Association for Computational
38
+ Linguistics (Volume 1: Long Papers), {ACL} 2022, Dublin, Ireland,
39
+ May 22-27, 2022},
40
+ pages = {320--335},
41
+ publisher = {Association for Computational Linguistics},
42
+ year = {2022},
43
+ }
44
+ ```
added_tokens.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<|endoftext|>": 50000,
3
+ "[SEP]": 50001,
4
+ "[CLS]": 50002,
5
+ "[MASK]": 50003,
6
+ "[UNUSED1]": 50004,
7
+ "[UNUSED2]": 50005,
8
+ "<|startofpiece|>": 50006,
9
+ "<|endofpiece|>": 50007,
10
+ "[sMASK]": 50008,
11
+ "[gMASK]": 50009
12
+ }
cog-pretrain.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ea6f4164152bc58d23e24e48f7bf4187aad72a32e97ec4b3acc832fe183cbc2
3
+ size 1021864
config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "THUDM/glm-large-chinese",
3
+ "architectures": [
4
+ "GLMModel"
5
+ ],
6
+ "attention_dropout_prob": 0.1,
7
+ "attention_scale": 1.0,
8
+ "auto_map": {
9
+ "AutoConfig": "configuration_glm.GLMConfig",
10
+ "AutoModel": "modeling_glm.GLMModel",
11
+ "AutoModelForSeq2SeqLM": "modeling_glm.GLMForConditionalGeneration",
12
+ "AutoModelForMultipleChoice": "modeling_glm.GLMForMultipleChoice"
13
+ },
14
+ "block_position_encoding": true,
15
+ "checkpoint_activations": false,
16
+ "checkpoint_num_layers": 1,
17
+ "embedding_dropout_prob": 0.1,
18
+ "hidden_size": 1024,
19
+ "initializer_range": 0.02,
20
+ "max_sequence_length": 1024,
21
+ "model_type": "glm",
22
+ "num_attention_heads": 16,
23
+ "num_layers": 24,
24
+ "output_dropout_prob": 0.1,
25
+ "output_predict": true,
26
+ "parallel_output": true,
27
+ "pool_token": "cls",
28
+ "relative_encoding": false,
29
+ "spell_func": "lstm",
30
+ "spell_length": null,
31
+ "torch_dtype": "float32",
32
+ "transformers_version": "4.23.1",
33
+ "vocab_size": 50048
34
+ }
configuration_glm.py ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2022 shunxing1234 and The HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """ GLM model configuration """
16
+
17
+ from transformers.configuration_utils import PretrainedConfig
18
+ from transformers.utils import logging
19
+
20
+ logger = logging.get_logger(__name__)
21
+
22
+ GLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {
23
+ "shunxing1234/GLM": "https://huggingface.co/shunxing1234/GLM/resolve/main/config.json",
24
+ # See all GLM models at https://huggingface.co/models?filter=glm
25
+ }
26
+
27
+
28
+ class GLMConfig(PretrainedConfig):
29
+ r"""
30
+ This is the configuration class to store the configuration of a [`~GLMModel`].
31
+ It is used to instantiate an GLM model according to the specified arguments, defining the model
32
+ architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
33
+ the GLM [shunxing1234/GLM-base-cased](https://huggingface.co/shunxing1234/GLM-base-cased) architecture.
34
+
35
+ Configuration objects inherit from [`PretrainedConfig`] and can be used
36
+ to control the model outputs. Read the documentation from [`PretrainedConfig`]
37
+ for more information.
38
+
39
+
40
+ Args:
41
+ vocab_size (`int`, *optional*, defaults to 30522):
42
+ Vocabulary size of the GLM model. Defines the number of different tokens that can be represented by the
43
+ `inputs_ids` passed when calling [`~GLMModel`] or
44
+ [`~TFGLMModel`].
45
+ hidden_size (`int`, *optional*, defaults to 768):
46
+ Dimension of the encoder layers and the pooler layer.
47
+ num_hidden_layers (`int`, *optional*, defaults to 12):
48
+ Number of hidden layers in the Transformer encoder.
49
+ num_attention_heads (`int`, *optional*, defaults to 12):
50
+ Number of attention heads for each attention layer in the Transformer encoder.
51
+ intermediate_size (`int`, *optional*, defaults to 3072):
52
+ Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
53
+ hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
54
+ The non-linear activation function (function or string) in the encoder and pooler.
55
+ If string, `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
56
+ hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
57
+ The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
58
+ attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
59
+ The dropout ratio for the attention probabilities.
60
+ max_position_embeddings (`int`, *optional*, defaults to 512):
61
+ The maximum sequence length that this model might ever be used with.
62
+ Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
63
+ type_vocab_size (`int`, *optional*, defaults to 2):
64
+ The vocabulary size of the `token_type_ids` passed when calling [`~GLMModel`] or
65
+ [`~TFGLMModel`].
66
+ initializer_range (`float`, *optional*, defaults to 0.02):
67
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
68
+ layer_norm_eps (`float`, *optional*, defaults to 1e-12):
69
+ The epsilon used by the layer normalization layers.
70
+ use_cache (`bool`, *optional*, defaults to `True`):
71
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
72
+ relevant if `config.is_decoder=True`.
73
+ Example:
74
+
75
+ ```python
76
+ >>> from transformers import GLMModel, GLMConfig
77
+
78
+ >>> # Initializing a GLM shunxing1234/GLM-base-cased style configuration
79
+ >>> configuration = GLMConfig()
80
+
81
+ >>> # Initializing a model from the shunxing1234/GLM-base-cased style configuration
82
+ >>> model = GLMModel(configuration)
83
+
84
+ >>> # Accessing the model configuration
85
+ >>> configuration = model.config
86
+ ```
87
+ """
88
+ model_type = "glm"
89
+ attribute_map = {
90
+ "num_hidden_layers": "num_layers"
91
+ }
92
+
93
+ def __init__(
94
+ self,
95
+ num_layers=24,
96
+ vocab_size=30592,
97
+ hidden_size=1024,
98
+ num_attention_heads=16,
99
+ embedding_dropout_prob=0.1,
100
+ attention_dropout_prob=0.1,
101
+ output_dropout_prob=0.1,
102
+ max_sequence_length=512,
103
+ checkpoint_activations=False,
104
+ checkpoint_num_layers=1,
105
+ parallel_output=True,
106
+ relative_encoding=False,
107
+ block_position_encoding=True,
108
+ output_predict=False,
109
+ spell_length=None,
110
+ spell_func="lstm",
111
+ attention_scale=1.0,
112
+ initializer_range=0.02,
113
+ pool_token="cls",
114
+ **kwargs
115
+ ):
116
+ self.num_layers = num_layers
117
+ self.vocab_size = vocab_size
118
+ self.hidden_size = hidden_size
119
+ self.num_attention_heads = num_attention_heads
120
+ self.embedding_dropout_prob = embedding_dropout_prob
121
+ self.attention_dropout_prob = attention_dropout_prob
122
+ self.output_dropout_prob = output_dropout_prob
123
+ self.max_sequence_length = max_sequence_length
124
+ self.checkpoint_activations = checkpoint_activations
125
+ self.checkpoint_num_layers = checkpoint_num_layers
126
+ self.parallel_output = parallel_output
127
+ self.relative_encoding = relative_encoding
128
+ self.block_position_encoding = block_position_encoding
129
+ self.output_predict = output_predict
130
+ self.spell_length = spell_length
131
+ self.spell_func = spell_func
132
+ self.attention_scale = attention_scale
133
+ self.initializer_range = initializer_range
134
+ self.pool_token = pool_token
135
+
136
+ super().__init__(**kwargs)
modeling_glm.py ADDED
@@ -0,0 +1,903 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2022 shunxing1234 The HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """ PyTorch GLM model. """
16
+
17
+ import math
18
+
19
+ import torch
20
+ import torch.utils.checkpoint
21
+ import torch.nn.functional as F
22
+ from torch.nn import init, LayerNorm, Linear, CrossEntropyLoss
23
+
24
+ from transformers.activations import gelu
25
+ from transformers.utils import (
26
+ add_code_sample_docstrings,
27
+ add_start_docstrings,
28
+ add_start_docstrings_to_model_forward,
29
+ )
30
+ from transformers.modeling_outputs import (
31
+ BaseModelOutputWithPastAndCrossAttentions,
32
+ ModelOutput,
33
+ )
34
+
35
+ from transformers.modeling_utils import (
36
+ PreTrainedModel,
37
+ )
38
+ from .configuration_glm import GLMConfig
39
+ from torch.nn.parameter import Parameter
40
+
41
+ _CHECKPOINT_FOR_DOC = "shunxing1234/GLM"
42
+ _CONFIG_FOR_DOC = "GLMConfig"
43
+ _TOKENIZER_FOR_DOC = "GLMTokenizer"
44
+
45
+ GLM_PRETRAINED_MODEL_ARCHIVE_LIST = [
46
+ "shunxing1234/GLM",
47
+ # See all GLM models at https://huggingface.co/models?filter=glm
48
+ ]
49
+
50
+
51
+ def unscaled_init_method(sigma):
52
+ """Init method based on N(0, sigma)."""
53
+
54
+ def init_(tensor):
55
+ return torch.nn.init.normal_(tensor, mean=0.0, std=sigma)
56
+
57
+ return init_
58
+
59
+
60
+ def scaled_init_method(mean, std, num_layers):
61
+ """Init method based on N(0, sigma/sqrt(2*num_layers)."""
62
+ std = std / math.sqrt(2.0 * num_layers)
63
+
64
+ def init_(tensor):
65
+ return torch.nn.init.normal_(tensor, mean=mean, std=std)
66
+
67
+ return init_
68
+
69
+
70
+ def ensure_divisibility(numerator, denominator):
71
+ """Ensure that numerator is divisible by the denominator."""
72
+ assert numerator % denominator == 0, '{} is not divisible by {}'.format(
73
+ numerator, denominator)
74
+
75
+
76
+ def divide(numerator, denominator):
77
+ """Ensure that numerator is divisible by the denominator and return
78
+ the division value."""
79
+ ensure_divisibility(numerator, denominator)
80
+ return numerator // denominator
81
+
82
+
83
+ def split_tensor_along_last_dim(tensor, num_partitions,
84
+ contiguous_split_chunks=False):
85
+ """Split a tensor along its last dimension.
86
+ Arguments:
87
+ tensor: input tensor.
88
+ num_partitions: number of partitions to split the tensor
89
+ contiguous_split_chunks: If True, make each chunk contiguous
90
+ in memory.
91
+ """
92
+ # Get the size and dimension.
93
+ last_dim = tensor.dim() - 1
94
+ last_dim_size = divide(tensor.size()[last_dim], num_partitions)
95
+ # Split.
96
+ tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
97
+ # Note: torch.split does not create contiguous tensors by default.
98
+ if contiguous_split_chunks:
99
+ return tuple(chunk.contiguous() for chunk in tensor_list)
100
+
101
+ return tensor_list
102
+
103
+
104
+ class MLP(torch.nn.Module):
105
+ """MLP for GPT2.
106
+
107
+ MLP will take the input with h hidden state, project it to 4*h
108
+ hidden dimension, perform gelu transformation, and project the
109
+ state back into h hidden dimension. At the end, dropout is also
110
+ applied.
111
+
112
+ Arguments:
113
+ hidden_size: The hidden size of the self attention.
114
+ output_dropout_prob: dropout probability for the outputs
115
+ after self attention and final output.
116
+ init_method: initialization method used for the weights. Note
117
+ that all biases are initialized to zero and
118
+ layernorm weight are initialized to one.
119
+ output_layer_init_method: output layer initialization. If None,
120
+ use `init_method`.
121
+ """
122
+
123
+ def __init__(self, hidden_size, output_dropout_prob, init_method,
124
+ output_layer_init_method=None):
125
+ super(MLP, self).__init__()
126
+ # Set output layer initialization if not provided.
127
+ if output_layer_init_method is None:
128
+ output_layer_init_method = init_method
129
+ # Project to 4h.
130
+ self.dense_h_to_4h = Linear(hidden_size, 4 * hidden_size)
131
+
132
+ # Project back to h.
133
+ self.dense_4h_to_h = Linear(
134
+ 4 * hidden_size,
135
+ hidden_size)
136
+
137
+ self.dropout = torch.nn.Dropout(output_dropout_prob)
138
+
139
+ def forward(self, hidden_states):
140
+ # [b, s, 4hp]
141
+ intermediate_parallel = self.dense_h_to_4h(hidden_states)
142
+ intermediate_parallel = gelu(intermediate_parallel)
143
+
144
+ # [b, s, h]
145
+ output = self.dense_4h_to_h(intermediate_parallel)
146
+ output = self.dropout(output)
147
+ return output
148
+
149
+
150
+ class VocabEmbedding(torch.nn.Module):
151
+ """Embedding parallelized in the vocabulary dimension.
152
+
153
+ This is mainly adapted from torch.nn.Embedding and all the default
154
+ values are kept.
155
+ Arguments:
156
+ num_embeddings: vocabulary size.
157
+ embedding_dim: size of hidden state.
158
+ init_method: method to initialize weights.
159
+ """
160
+
161
+ def __init__(self, config):
162
+ super(VocabEmbedding, self).__init__()
163
+ # Keep the input dimensions.
164
+ self.num_embeddings = config.vocab_size
165
+ self.embedding_dim = config.hidden_size
166
+ # Set the detauls for compatibility.
167
+ self.padding_idx = None
168
+ self.max_norm = None
169
+ self.norm_type = 2.
170
+ self.scale_grad_by_freq = False
171
+ self.sparse = False
172
+ self._weight = None
173
+
174
+ self.vocab_start_index = 0
175
+ self.vocab_end_index = self.num_embeddings
176
+
177
+ # Allocate weights.
178
+ self.weight = Parameter(torch.Tensor(self.num_embeddings,
179
+ self.embedding_dim))
180
+ # And initialize.
181
+ init.xavier_normal_(self.weight)
182
+
183
+ def forward(self, input_):
184
+ # Get the embeddings.
185
+ output = F.embedding(input_, self.weight,
186
+ self.padding_idx, self.max_norm,
187
+ self.norm_type, self.scale_grad_by_freq,
188
+ self.sparse)
189
+ return output
190
+
191
+
192
+ class PositionalEmbedding(torch.nn.Module):
193
+
194
+ def __init__(self, hidden_size):
195
+ super(PositionalEmbedding, self).__init__()
196
+
197
+ self.hidden_size = hidden_size
198
+
199
+ inv_freq = 1 / (10000 ** (torch.arange(0.0, hidden_size, 2.0) / hidden_size))
200
+ self.register_buffer('inv_freq', inv_freq)
201
+
202
+ def forward(self, pos_seq, bsz=None):
203
+ sinusoid_inp = torch.ger(pos_seq, self.inv_freq)
204
+ pos_emb = torch.cat([sinusoid_inp.sin(), sinusoid_inp.cos()], dim=-1)
205
+
206
+ if bsz is not None:
207
+ return pos_emb[None, :, :].expand(bsz, -1, -1)
208
+ else:
209
+ return pos_emb[None, :, :]
210
+
211
+
212
+ class SelfAttention(torch.nn.Module):
213
+ """self-attention layer for GLM.
214
+
215
+ Self-attention layer takes input with size [b, s, h] where b is
216
+ the batch size, s is the sequence lenght, and h is the hidden size
217
+ and creates output of the same size.
218
+ Arguments:
219
+ hidden_size: total hidden size of the layer (h).
220
+ num_attention_heads: number of attention heads (n). Note that we
221
+ require n to be divisible by number of GPUs
222
+ used to parallelize the model. Also, we
223
+ require hidden size to be divisible by n.
224
+ attention_dropout_prob: dropout probability for the attention scores.
225
+ init_method: weight initialization.
226
+ output_layer_init_method: output layer initialization. If None, use
227
+ `init_method`.
228
+ We use the following notation:
229
+ h: hidden_size
230
+ n: num_attention_heads
231
+ p: number of partitions
232
+ np: n/p
233
+ hp: h/p
234
+ hn: h/n
235
+ b: batch size
236
+ s: sequence length
237
+ """
238
+
239
+ def __init__(self, hidden_size, num_attention_heads,
240
+ attention_dropout_prob, output_dropout_prob,
241
+ init_method, output_layer_init_method=None,
242
+ attention_scale=1.0):
243
+ super(SelfAttention, self).__init__()
244
+ # Set output layer initialization if not provided.
245
+ if output_layer_init_method is None:
246
+ output_layer_init_method = init_method
247
+ # Per attention head and per partition values.
248
+ self.hidden_size = hidden_size
249
+ self.hidden_size_per_attention_head = divide(hidden_size,
250
+ num_attention_heads)
251
+
252
+ self.num_attention_heads = num_attention_heads
253
+ self.attention_scale = attention_scale
254
+ # Strided linear layer.
255
+ self.query_key_value = Linear(hidden_size, 3 * hidden_size)
256
+
257
+ # Dropout. Note that for a single iteration, this layer will generate
258
+ # different outputs on different number of parallel partitions but
259
+ # on average it should not be partition dependent.
260
+ self.attention_dropout = torch.nn.Dropout(attention_dropout_prob)
261
+
262
+ # Output.
263
+ self.dense = Linear(hidden_size,
264
+ hidden_size)
265
+ self.output_dropout = torch.nn.Dropout(output_dropout_prob)
266
+
267
+ def _transpose_for_scores(self, tensor):
268
+ """Transpose a 3D tensor [b, s, np*hn] into a 4D tensor with
269
+ size [b, np, s, hn].
270
+ """
271
+ new_tensor_shape = tensor.size()[:-1] + \
272
+ (self.num_attention_heads,
273
+ self.hidden_size_per_attention_head)
274
+ tensor = tensor.view(*new_tensor_shape)
275
+ return tensor.permute(0, 2, 1, 3)
276
+
277
+ def forward(self, hidden_states, ltor_mask, mem=None):
278
+ # hidden_states: [b, s, h]
279
+ # ltor_mask: [b,1,s,s]
280
+
281
+ # Attention heads. [b, s, hp]
282
+ query_length = hidden_states.size(1)
283
+ # self attention
284
+ if mem is None:
285
+ mixed_x_layer = self.query_key_value(hidden_states)
286
+ (mixed_query_layer,
287
+ mixed_key_layer,
288
+ mixed_value_layer) = split_tensor_along_last_dim(mixed_x_layer, 3)
289
+ else:
290
+ cat = torch.cat((mem, hidden_states), 1)
291
+ mixed_x_layer = self.query_key_value(cat)
292
+ (mixed_query_layer,
293
+ mixed_key_layer,
294
+ mixed_value_layer) = split_tensor_along_last_dim(mixed_x_layer, 3)
295
+ mixed_query_layer = mixed_query_layer[:, -query_length:]
296
+
297
+ # Reshape and transpose [b, np, s, hn]
298
+ query_layer = self._transpose_for_scores(mixed_query_layer)
299
+ key_layer = self._transpose_for_scores(mixed_key_layer)
300
+ value_layer = self._transpose_for_scores(mixed_value_layer)
301
+
302
+ if self.attention_scale > 1.0:
303
+ # Raw attention scores. [b, np, s, s]
304
+ attention_scores = torch.matmul(query_layer / math.sqrt(self.attention_scale),
305
+ key_layer.transpose(-1, -2) / math.sqrt(
306
+ self.hidden_size_per_attention_head * self.attention_scale))
307
+ else:
308
+ attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2) / math.sqrt(
309
+ self.hidden_size_per_attention_head))
310
+
311
+ # Apply the left to right attention mask.
312
+ ltor_mask = ltor_mask.type_as(attention_scores)
313
+ attention_scores = torch.mul(attention_scores, ltor_mask)
314
+ if self.attention_scale > 1.0:
315
+ max_attention_scores = attention_scores.max(dim=-1, keepdim=True)[0]
316
+ attention_scores -= max_attention_scores
317
+ attention_scores *= self.attention_scale
318
+
319
+ attention_scores = attention_scores + (-65504.0) * (1.0 - ltor_mask)
320
+ # Attention probabilities. [b, np, s, s]
321
+ attention_probs = torch.nn.Softmax(dim=-1)(attention_scores)
322
+ # This is actually dropping out entire tokens to attend to, which might
323
+ # seem a bit unusual, but is taken from the original Transformer paper.
324
+ # with get_cuda_rng_tracker().fork():
325
+ attention_probs = self.attention_dropout(attention_probs)
326
+
327
+ # Context layer.
328
+ # [b, np, s, hn]
329
+ context_layer = torch.matmul(attention_probs, value_layer)
330
+ # [b, s, np, hn]
331
+ context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
332
+ new_context_layer_shape = context_layer.size()[:-2] + \
333
+ (self.hidden_size,)
334
+ # [b, s, hp]
335
+ context_layer = context_layer.view(*new_context_layer_shape)
336
+
337
+ # Output. [b, s, h]
338
+ output = self.dense(context_layer)
339
+ output = self.output_dropout(output)
340
+
341
+ return output
342
+
343
+
344
+ class GLMBlock(torch.nn.Module):
345
+ """A single layer transformer for GLM.
346
+
347
+ We use the following notation:
348
+ h: hidden size
349
+ n: number of attention heads
350
+ b: batch size
351
+ s: sequence length
352
+ Transformore layer takes input with size [b, s, h] and returns an
353
+ output of the same size.
354
+
355
+ Arguments:
356
+ hidden_size: The hidden size of the self attention.
357
+ num_attention_heads: number of attention head in the self
358
+ attention.
359
+ attention_dropout_prob: dropout probability of the attention
360
+ score in self attention.
361
+ output_dropout_prob: dropout probability for the outputs
362
+ after self attention and final output.
363
+ layernorm_epsilon: epsilon used in layernorm to avoid
364
+ division by zero.
365
+ init_method: initialization method used for the weights. Note
366
+ that all biases are initialized to zero and
367
+ layernorm weight are initialized to one.
368
+ output_layer_init_method: output layers (attention output and
369
+ mlp output) initialization. If None,
370
+ use `init_method`.
371
+ """
372
+
373
+ def __init__(self,
374
+ hidden_size,
375
+ num_attention_heads,
376
+ attention_dropout_prob,
377
+ output_dropout_prob,
378
+ layernorm_epsilon,
379
+ init_method,
380
+ output_layer_init_method=None,
381
+ attention_scale=1.0):
382
+ super(GLMBlock, self).__init__()
383
+ # Set output layer initialization if not provided.
384
+ if output_layer_init_method is None:
385
+ output_layer_init_method = init_method
386
+
387
+ # Layernorm on the input data.
388
+ self.input_layernorm = LayerNorm(hidden_size, eps=layernorm_epsilon)
389
+
390
+ # Self attention.
391
+ self.attention = SelfAttention(
392
+ hidden_size,
393
+ num_attention_heads,
394
+ attention_dropout_prob,
395
+ output_dropout_prob,
396
+ init_method,
397
+ output_layer_init_method=output_layer_init_method,
398
+ attention_scale=attention_scale)
399
+
400
+ # Layernorm on the input data.
401
+ self.post_attention_layernorm = LayerNorm(hidden_size,
402
+ eps=layernorm_epsilon)
403
+
404
+ # MLP
405
+ self.mlp = MLP(
406
+ hidden_size,
407
+ output_dropout_prob,
408
+ init_method,
409
+ output_layer_init_method=output_layer_init_method)
410
+
411
+ def forward(self, hidden_states, ltor_mask, mem=None):
412
+ # hidden_states: [b, s, h]
413
+ # ltor_mask: [b,1, s,s]
414
+
415
+ # Layer norm at the begining of the transformer layer.
416
+ layernorm_output = self.input_layernorm(hidden_states)
417
+ mem = self.input_layernorm(mem) if mem is not None else None
418
+ # Self attention.
419
+ attention_output = self.attention(layernorm_output, ltor_mask, mem)
420
+ # Residual connection.
421
+ layernorm_input = hidden_states + attention_output
422
+ # Layer norm post the self attention.
423
+ layernorm_output = self.post_attention_layernorm(layernorm_input)
424
+ # MLP.
425
+ mlp_output = self.mlp(layernorm_output)
426
+ # Second residual connection.
427
+ output = layernorm_input + mlp_output
428
+
429
+ return output
430
+
431
+
432
+ class GLMStack(torch.nn.Module):
433
+ """GLM transformer.
434
+
435
+ This module takes input from embedding layer and it's output can
436
+ be used directly by a logit layer. It consists of L (num-layers)
437
+ blocks of:
438
+ layer norm
439
+ self attention
440
+ residual connection
441
+ layer norm
442
+ mlp
443
+ residual connection
444
+ followed by a final layer norm.
445
+
446
+ Arguments:
447
+ num_layers: Number of transformer layers.
448
+ hidden_size: The hidden size of the self attention.
449
+ num_attention_heads: number of attention head in the self
450
+ attention.
451
+ attention_dropout_prob: dropout probability of the attention
452
+ score in self attention.
453
+ output_dropout_prob: dropout probability for the outputs
454
+ after self attention and final output.
455
+ checkpoint_activations: if True, checkpoint activations.
456
+ checkpoint_num_layers: number of layers to checkpoint. This
457
+ is basically the chunk size in checkpoitning.
458
+ layernorm_epsilon: epsilon used in layernorm to avoid
459
+ division by zero.
460
+ init_method_std: standard deviation of the init method which has
461
+ the form N(0, std).
462
+ use_scaled_init_for_output_weights: If Ture use 1/sqrt(2*num_layers)
463
+ scaling for the output weights (
464
+ output of self attention and mlp).
465
+ """
466
+
467
+ def __init__(self,
468
+ num_layers,
469
+ hidden_size,
470
+ num_attention_heads,
471
+ max_sequence_length,
472
+ embedding_dropout_prob,
473
+ attention_dropout_prob,
474
+ output_dropout_prob,
475
+ checkpoint_activations,
476
+ checkpoint_num_layers=1,
477
+ layernorm_epsilon=1.0e-5,
478
+ init_method_std=0.02,
479
+ use_scaled_init_for_output_weights=True,
480
+ block_position_encoding=False,
481
+ attention_scale=1.0,
482
+ ):
483
+ super(GLMStack, self).__init__()
484
+ self.hidden_size = hidden_size
485
+ # Store activation checkpoiting flag.
486
+ self.checkpoint_activations = checkpoint_activations
487
+ self.checkpoint_num_layers = checkpoint_num_layers
488
+
489
+ output_layer_init_method = None
490
+ if use_scaled_init_for_output_weights:
491
+ output_layer_init_method = scaled_init_method(0.0, init_method_std,
492
+ num_layers)
493
+ # Embeddings dropout
494
+ self.embedding_dropout = torch.nn.Dropout(embedding_dropout_prob)
495
+ self.block_position_encoding = block_position_encoding
496
+
497
+ # Position embedding (serial).
498
+ if block_position_encoding:
499
+ self.position_embeddings = torch.nn.Embedding(max_sequence_length + 1, hidden_size)
500
+ self.block_position_embeddings = torch.nn.Embedding(max_sequence_length + 1, hidden_size)
501
+ torch.nn.init.normal_(self.block_position_embeddings.weight, mean=0.0, std=init_method_std)
502
+ else:
503
+ self.position_embeddings = torch.nn.Embedding(max_sequence_length, hidden_size)
504
+ # Initialize the position embeddings.
505
+ torch.nn.init.normal_(self.position_embeddings.weight, mean=0.0, std=init_method_std)
506
+
507
+ def get_layer():
508
+
509
+ return GLMBlock(
510
+ hidden_size,
511
+ num_attention_heads,
512
+ attention_dropout_prob,
513
+ output_dropout_prob,
514
+ layernorm_epsilon,
515
+ unscaled_init_method(init_method_std),
516
+ output_layer_init_method=output_layer_init_method,
517
+ attention_scale=attention_scale)
518
+
519
+ # Transformer layers.
520
+ self.layers = torch.nn.ModuleList(
521
+ [get_layer() for _ in range(num_layers)])
522
+
523
+ # Final layer norm before output.
524
+ self.final_layernorm = LayerNorm(hidden_size, eps=layernorm_epsilon)
525
+
526
+ def forward(self, hidden_states, position_ids, attention_mask, memory_states=None):
527
+
528
+ batch_size, query_length = hidden_states.size()[:2]
529
+ memory_length = memory_states[0].size(1) if memory_states else 0
530
+ # attention mask is the beginning postion of B region, \in [0, query_len)
531
+ is_scalar = torch.numel(attention_mask) == 1
532
+ is_sep = is_scalar or torch.numel(attention_mask) == batch_size
533
+ if is_sep:
534
+ sep = attention_mask.item() if is_scalar else attention_mask
535
+
536
+ # conventional transformer
537
+ def build_mask_matrix(seq_length, sep, memory_length=0):
538
+ m = hidden_states.new_ones((1, seq_length, seq_length))
539
+ m = torch.tril(m)
540
+ if is_scalar:
541
+ m[0, :, :int(sep)] = 1
542
+ else:
543
+ m = m.expand(batch_size, -1, -1)
544
+ ids = torch.arange(seq_length, device=sep.device, dtype=sep.dtype).view(1, -1)
545
+ mask = ids < sep.view(-1, 1)
546
+ m = m.masked_fill(mask.unsqueeze(1).expand_as(m), 1)
547
+ if memory_length > 0:
548
+ m = m.expand(batch_size, -1, -1)
549
+ m = torch.cat((hidden_states.new_ones((batch_size, seq_length, memory_length)), m), dim=2)
550
+ m = m.unsqueeze(1)
551
+ return m
552
+
553
+ attention_mask = build_mask_matrix(query_length, sep, memory_length=memory_length)
554
+ else:
555
+ if attention_mask.dim() == 2:
556
+ attention_mask = attention_mask.unsqueeze(1).unsqueeze(1)
557
+ attention_mask = attention_mask[:, :, :, -query_length - memory_length:]
558
+
559
+ if self.block_position_encoding:
560
+ position_ids, block_position_ids = position_ids[:, 0], position_ids[:, 1]
561
+ position_embeddings = self.position_embeddings(position_ids)
562
+
563
+ hidden_states = hidden_states + position_embeddings
564
+ if self.block_position_encoding:
565
+ block_position_embeddings = self.block_position_embeddings(block_position_ids)
566
+ hidden_states = hidden_states + block_position_embeddings
567
+ hidden_states = self.embedding_dropout(hidden_states)
568
+
569
+ def check_detach(_hidden_states):
570
+ return _hidden_states.detach()
571
+
572
+ mem_layers = [check_detach(hidden_states)]
573
+
574
+ for i, layer in enumerate(self.layers):
575
+
576
+ args = [hidden_states, attention_mask]
577
+
578
+ def create_custom_forward(module):
579
+ def custom_forward(*inputs):
580
+ # None for past_key_value
581
+ return module(*inputs)
582
+
583
+ return custom_forward
584
+
585
+ mem_i = memory_states[i] if memory_states else None
586
+
587
+ if self.checkpoint_activations:
588
+ hidden_states = torch.utils.checkpoint.checkpoint(
589
+ create_custom_forward(layer),
590
+ hidden_states,
591
+ mem=mem_i,
592
+ )
593
+ else:
594
+ hidden_states = layer(*args, mem=mem_i)
595
+ mem_layers.append(check_detach(hidden_states))
596
+
597
+ # Final layer norm.
598
+ output = self.final_layernorm(hidden_states)
599
+ mem_layers = self.update_mems(mem_layers, memory_states)
600
+ return (output, mem_layers)
601
+
602
+ def update_mems(self, hiddens, mems):
603
+ memory_length = mems[0].size(1) if mems else 0
604
+ query_length = hiddens[0].size(1)
605
+ new_memory_length = memory_length + query_length
606
+
607
+ new_mems = []
608
+ # with torch.no_grad():
609
+ for i in range(len(hiddens)):
610
+ if new_memory_length <= query_length:
611
+ new_mems.append(hiddens[i][:, -new_memory_length:])
612
+ else:
613
+ new_mems.append(torch.cat((mems[i][:, -new_memory_length + query_length:], hiddens[i]), dim=1))
614
+ return new_mems
615
+
616
+
617
+ class GLMPreTrainedModel(PreTrainedModel):
618
+ """
619
+ An abstract class to handle weights initialization and
620
+ a simple interface for downloading and loading pretrained models.
621
+ """
622
+
623
+ config_class = GLMConfig
624
+ base_model_prefix = "glm"
625
+ supports_gradient_checkpointing = True
626
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
627
+
628
+ def _init_weights(self, module):
629
+ """ Initialize the weights """
630
+ if isinstance(module, torch.nn.Linear):
631
+ # Slightly different from the TF version which uses truncated_normal for initialization
632
+ # cf https://github.com/pytorch/pytorch/pull/5617
633
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
634
+ if module.bias is not None:
635
+ module.bias.data.zero_()
636
+ elif isinstance(module, torch.nn.Embedding):
637
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
638
+ if module.padding_idx is not None:
639
+ module.weight.data[module.padding_idx].zero_()
640
+ elif isinstance(module, torch.nn.LayerNorm):
641
+ module.bias.data.zero_()
642
+ module.weight.data.fill_(1.0)
643
+
644
+ def _set_gradient_checkpointing(self, module, value=False):
645
+ if isinstance(module, GLMModel):
646
+ module.gradient_checkpointing = value
647
+
648
+
649
+ GLM_START_DOCSTRING = r"""
650
+ This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class.
651
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general
652
+ usage and behavior.
653
+
654
+ Parameters:
655
+ config ([`~GLMConfig`]): Model configuration class with all the parameters of the model.
656
+ Initializing with a config file does not load the weights associated with the model, only the configuration.
657
+ Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
658
+ """
659
+
660
+ GLM_INPUTS_DOCSTRING = r"""
661
+ Args:
662
+ input_ids (`torch.LongTensor` of shape `({0})`):
663
+ Indices of input sequence tokens in the vocabulary.
664
+
665
+ Indices can be obtained using [`GLMTokenizer`].
666
+ See [`PreTrainedTokenizer.encode`] and
667
+ [`PreTrainedTokenizer.__call__`] for details.
668
+
669
+ [What are input IDs?](../glossary#input-ids)
670
+ attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
671
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
672
+
673
+ - 1 for tokens that are **not masked**,
674
+ - 0 for tokens that are **masked**.
675
+
676
+ [What are attention masks?](../glossary#attention-mask)
677
+ token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
678
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
679
+
680
+ - 0 corresponds to a *sentence A* token,
681
+ - 1 corresponds to a *sentence B* token.
682
+
683
+ [What are token type IDs?](../glossary#token-type-ids)
684
+ position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
685
+ Indices of positions of each input sequence tokens in the position embeddings.
686
+ Selected in the range `[0, config.max_position_embeddings - 1]`.
687
+
688
+ [What are position IDs?](../glossary#position-ids)
689
+ head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
690
+ Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
691
+
692
+ - 1 indicates the head is **not masked**,
693
+ - 0 indicates the head is **masked**.
694
+
695
+ inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
696
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
697
+ This is useful if you want more control over how to convert *input_ids* indices into associated vectors
698
+ than the model's internal embedding lookup matrix.
699
+ output_attentions (`bool`, *optional*):
700
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
701
+ tensors for more detail.
702
+ output_hidden_states (`bool`, *optional*):
703
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
704
+ more detail.
705
+ return_dict (`bool`, *optional*):
706
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
707
+ """
708
+
709
+
710
+ @add_start_docstrings(
711
+ "The bare GLM Model transformer outputting raw hidden-states without any specific head on top.",
712
+ GLM_START_DOCSTRING,
713
+ )
714
+ class GLMModel(GLMPreTrainedModel):
715
+ """
716
+
717
+ The model can behave as an encoder (with only self-attention) as well
718
+ as a decoder, in which case a layer of cross-attention is added between
719
+ the self-attention layers, following the architecture described in [Attention is
720
+ all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani,
721
+ Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
722
+
723
+ To behave as an decoder the model needs to be initialized with the
724
+ `is_decoder` argument of the configuration set to `True`.
725
+ To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder`
726
+ argument and `add_cross_attention` set to `True`; an
727
+ `encoder_hidden_states` is then expected as an input to the forward pass.
728
+ """
729
+
730
+ def __init__(self, config):
731
+ super().__init__(config)
732
+ self.config = config
733
+ self.output_predict = config.output_predict
734
+ # Word embeddings (parallel).
735
+ self.word_embeddings = VocabEmbedding(config)
736
+
737
+ # Transformer
738
+ self.transformer = GLMStack(config.num_layers,
739
+ config.hidden_size,
740
+ config.num_attention_heads,
741
+ config.max_sequence_length,
742
+ config.embedding_dropout_prob,
743
+ config.attention_dropout_prob,
744
+ config.output_dropout_prob,
745
+ config.checkpoint_activations,
746
+ config.checkpoint_num_layers,
747
+ attention_scale=config.attention_scale,
748
+ block_position_encoding=config.block_position_encoding)
749
+
750
+ # Initialize weights and apply final processing
751
+ self.post_init()
752
+
753
+ @add_start_docstrings_to_model_forward(GLM_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
754
+ @add_code_sample_docstrings(
755
+ processor_class=_TOKENIZER_FOR_DOC,
756
+ checkpoint=_CHECKPOINT_FOR_DOC,
757
+ output_type=BaseModelOutputWithPastAndCrossAttentions,
758
+ config_class=_CONFIG_FOR_DOC,
759
+ )
760
+ def forward(
761
+ self,
762
+ input_ids=None,
763
+ position_ids=None,
764
+ attention_mask=None,
765
+ mems=None,
766
+ **kwargs
767
+ ):
768
+ batch_size = input_ids.size(0)
769
+ words_embeddings = self.word_embeddings(input_ids)
770
+ embeddings = words_embeddings
771
+
772
+ device = input_ids.device
773
+ input_shape = input_ids.size()
774
+
775
+ if position_ids is None:
776
+ position_ids = torch.arange(0, input_shape[-1], dtype=torch.long, device=device)
777
+ block_position_ids = torch.zeros(input_shape[-1], dtype=torch.long, device=device)
778
+ position_ids = torch.stack((position_ids, block_position_ids), dim=0).unsqueeze(0)
779
+ if attention_mask is None:
780
+ attention_mask = torch.zeros(batch_size)
781
+ # Transformer.
782
+ transformer_output = self.transformer(embeddings, position_ids, attention_mask, mems)
783
+ logits, hidden_layers = transformer_output
784
+ # outputs = hidden_layers
785
+ if self.output_predict:
786
+ # Parallel logits.
787
+ # logits_parallel = mpu.copy_to_model_parallel_region(
788
+ # logits)
789
+ logits = F.linear(logits, self.word_embeddings.weight)
790
+
791
+ return ModelOutput(
792
+ logits=logits,
793
+ mems=hidden_layers,
794
+ )
795
+
796
+
797
+ @add_start_docstrings(
798
+ """GLM Model transformer for multiple choice classification""",
799
+ GLM_START_DOCSTRING
800
+ )
801
+ class GLMForMultipleChoice(GLMPreTrainedModel):
802
+ def __init__(self, config):
803
+ super().__init__(config)
804
+ self.glm = GLMModel(config)
805
+ self.post_init()
806
+
807
+ def forward(
808
+ self,
809
+ input_ids=None,
810
+ position_ids=None,
811
+ attention_mask=None,
812
+ choice_ids=None,
813
+ choice_indices=None,
814
+ labels=None,
815
+ mems=None,
816
+ **kwargs
817
+ ):
818
+ model_output = self.glm.forward(input_ids, position_ids, attention_mask, mems=mems, **kwargs)
819
+ lm_logits = model_output.logits
820
+ log_probs = []
821
+ for output, choices, choice_index in zip(F.log_softmax(lm_logits, dim=-1), choice_ids, choice_indices):
822
+ log_probs_single = []
823
+ for choice, choice_target_id in zip(choices, choice_index):
824
+ tmp = output[choice_target_id, choice]
825
+ log_probs_single.append(tmp.sum())
826
+ log_probs.append(torch.stack(log_probs_single))
827
+ log_probs = torch.stack(log_probs)
828
+ loss = None
829
+ if labels is not None:
830
+ loss_fct = CrossEntropyLoss()
831
+ loss = loss_fct(log_probs, labels)
832
+ return ModelOutput(
833
+ loss=loss,
834
+ logits=log_probs,
835
+ lm_logits=lm_logits,
836
+ mems=model_output.mems
837
+ )
838
+
839
+ @add_start_docstrings(
840
+ """GLM Model transformer with a `language modeling` head on top""",
841
+ GLM_START_DOCSTRING,
842
+ )
843
+ class GLMForConditionalGeneration(GLMPreTrainedModel):
844
+ def __init__(self, config):
845
+ super().__init__(config)
846
+ self.glm = GLMModel(config)
847
+ self.post_init()
848
+
849
+ def _reorder_cache(self, past, beam_idx):
850
+ # if decoder past is not included in output
851
+ # speedy decoding is disabled and no need to reorder
852
+ if past is None:
853
+ return past
854
+ reordered_decoder_past = ()
855
+ for layer_past_states in past:
856
+ # get the correct batch idx from layer past batch dim
857
+ reordered_decoder_past = reordered_decoder_past + (
858
+ layer_past_states.index_select(0, beam_idx.to(layer_past_states.device)),)
859
+ return reordered_decoder_past
860
+
861
+ def prepare_inputs_for_generation(self, input_ids, past=None, position_ids=None, generation_attention_mask=None,
862
+ **kwargs):
863
+ # only last token for inputs_ids if past is defined in kwargs
864
+ attention_mask = generation_attention_mask
865
+ seq_length = input_ids.shape[1]
866
+ if past:
867
+ if position_ids is not None:
868
+ position_ids = position_ids[:, :, seq_length - 1].unsqueeze(-1)
869
+ if attention_mask is not None:
870
+ attention_mask = attention_mask[:, :, seq_length - 1, :seq_length].unsqueeze(-2)
871
+ input_ids = input_ids[:, -1].unsqueeze(-1)
872
+ else:
873
+ if position_ids is not None:
874
+ position_ids = position_ids[:, :, :seq_length]
875
+ if attention_mask is not None:
876
+ attention_mask = attention_mask[:, :, :seq_length, :seq_length]
877
+ return {
878
+ "input_ids": input_ids,
879
+ "position_ids": position_ids,
880
+ "attention_mask": attention_mask,
881
+ "mems": past,
882
+ }
883
+
884
+ def forward(
885
+ self,
886
+ input_ids=None,
887
+ position_ids=None,
888
+ attention_mask=None,
889
+ labels=None,
890
+ mems=None,
891
+ **kwargs
892
+ ):
893
+ model_output = self.glm.forward(input_ids, position_ids, attention_mask, mems=mems, **kwargs)
894
+ lm_logits = model_output.logits
895
+ loss = None
896
+ if labels is not None:
897
+ loss_fct = CrossEntropyLoss(ignore_index=-100)
898
+ loss = loss_fct(lm_logits.view(-1, lm_logits.size(-1)), labels.view(-1))
899
+ return ModelOutput(
900
+ loss=loss,
901
+ logits=lm_logits,
902
+ mems=model_output.mems
903
+ )
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d797cfd353cde41644b62fef5badb36e8b4a0abfa262360401b61caaa73234b
3
+ size 711378663
tokenization_glm.py ADDED
@@ -0,0 +1,362 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from typing import Optional, Tuple, List, Union
3
+ from shutil import copyfile
4
+ import torch
5
+
6
+ from transformers import PreTrainedTokenizer, RobertaTokenizer, GPT2Tokenizer, BertTokenizer
7
+ from transformers.utils import logging
8
+ from transformers.tokenization_utils_base import BatchEncoding
9
+ from transformers.models.auto.tokenization_auto import get_tokenizer_config
10
+ from transformers.utils.generic import _is_torch_device
11
+ import sentencepiece as spm
12
+
13
+ logger = logging.get_logger(__name__)
14
+
15
+
16
+ class GLMBatchEncoding(BatchEncoding):
17
+ def to(self, device: Union[str, "torch.device"]) -> "BatchEncoding":
18
+ """
19
+ Send all values to device by calling `v.to(device)` (PyTorch only).
20
+
21
+ Args:
22
+ device (`str` or `torch.device`): The device to put the tensors on.
23
+
24
+ Returns:
25
+ [`BatchEncoding`]: The same instance after modification.
26
+ """
27
+
28
+ # This check catches things like APEX blindly calling "to" on all inputs to a module
29
+ # Otherwise it passes the casts down and casts the LongTensor containing the token idxs
30
+ # into a HalfTensor
31
+ if isinstance(device, str) or _is_torch_device(device) or isinstance(device, int):
32
+ self.data = {k: v.to(device=device) if torch.is_tensor(v) else v for k, v in self.data.items()}
33
+ else:
34
+ logger.warning(f"Attempting to cast a BatchEncoding to type {str(device)}. This is not supported.")
35
+ return self
36
+
37
+
38
+ class GLMTokenizerMixin:
39
+ @property
40
+ def sop_token(self) -> Optional[str]:
41
+ return "<|startofpiece|>"
42
+
43
+ @property
44
+ def sop_token_id(self) -> Optional[int]:
45
+ """
46
+ `Optional[int]`: Id of the start token in the vocabulary, used when training a model with autoregressive blank filling.
47
+ """
48
+ return self.convert_tokens_to_ids(self.sop_token)
49
+
50
+ @property
51
+ def eop_token(self) -> Optional[str]:
52
+ return "<|endofpiece|>"
53
+
54
+ @property
55
+ def eop_token_id(self) -> Optional[int]:
56
+ """
57
+ `Optional[int]`: Id of the end token in the vocabulary, used when training a model with autoregressive blank filling.
58
+ """
59
+ return self.convert_tokens_to_ids(self.eop_token)
60
+
61
+ @property
62
+ def gmask_token_id(self) -> int:
63
+ return self.convert_tokens_to_ids("[gMASK]")
64
+
65
+ @property
66
+ def smask_token_id(self) -> int:
67
+ return self.convert_tokens_to_ids("[sMASK]")
68
+
69
+ @property
70
+ def mask_token_ids(self):
71
+ return [self.mask_token_id, self.smask_token_id, self.gmask_token_id]
72
+
73
+ def _build_input_for_multiple_choice(self, context, choices):
74
+ context_id = context["input_ids"]
75
+ if torch.is_tensor(context_id):
76
+ context_id = context_id.tolist()
77
+
78
+ division = len(context_id)
79
+ mask_position = context_id.index(self.mask_token_id)
80
+
81
+ token = torch.tensor(context_id, dtype=torch.long)
82
+ attention_mask = [context["attention_mask"].expand(division, -1)]
83
+ position_id = torch.arange(division, dtype=torch.long)
84
+ block_position_id = torch.zeros(division, dtype=torch.long)
85
+
86
+ choice_ids, choice_indices = [], []
87
+
88
+ for choice_str in choices:
89
+ choice = torch.tensor(self(choice_str, add_special_tokens=False, padding=False)['input_ids'],
90
+ dtype=torch.long)
91
+ choice_ids.append(choice)
92
+ choice_indices.append(torch.arange(len(token), len(token) + len(choice), dtype=torch.long))
93
+ attention_mask.append(torch.tril(torch.ones((len(choice), len(choice)), dtype=torch.long)))
94
+
95
+ token = torch.cat((token, torch.tensor([self.sop_token_id], dtype=torch.long), choice[:-1]))
96
+ position_id = torch.cat((position_id, torch.tensor([mask_position] * len(choice), dtype=torch.long)))
97
+ block_position_id = torch.cat((block_position_id, torch.arange(1, 1 + len(choice), dtype=torch.long)))
98
+
99
+ attention_mask = torch.block_diag(*attention_mask)
100
+ attention_mask[division:, :division] = context["attention_mask"].unsqueeze(0)
101
+
102
+ return {
103
+ "input_ids": token,
104
+ "position_ids": torch.stack((position_id, block_position_id)),
105
+ "attention_mask": attention_mask,
106
+ "choice_ids": choice_ids,
107
+ "choice_indices": choice_indices
108
+ }
109
+
110
+ def _pad_batch(self, tokens, position_ids, attention_mask, max_seq_length):
111
+ pad_length = max_seq_length - len(tokens)
112
+ attention_mask = torch.nn.functional.pad(
113
+ attention_mask,
114
+ (0, pad_length, 0, pad_length),
115
+ mode="constant",
116
+ value=0,
117
+ )
118
+ tokens = torch.cat((tokens, torch.zeros(pad_length, dtype=torch.long)))
119
+ position_ids = torch.cat((position_ids, position_ids[..., -1:].expand(-1, pad_length)), dim=-1)
120
+ return tokens, position_ids, attention_mask
121
+
122
+ def _collate(self, samples):
123
+ TILE = 1
124
+ length_to_pad = (max(map(lambda spl: len(spl["input_ids"]), samples)) + TILE - 1) // TILE * TILE
125
+
126
+ token_batch, position_id_batch, attention_mask_batch = [], [], []
127
+ choices_batch, choice_target_ids_batch = [], []
128
+
129
+ for sample in samples:
130
+ token, position_id, attention_mask = self._pad_batch(
131
+ sample["input_ids"], sample["position_ids"], sample["attention_mask"], length_to_pad
132
+ )
133
+ token_batch.append(token)
134
+ position_id_batch.append(position_id)
135
+ attention_mask_batch.append(attention_mask)
136
+ choices_batch.append(sample["choice_ids"])
137
+ choice_target_ids_batch.append(sample["choice_indices"])
138
+ return {
139
+ "input_ids": torch.stack(token_batch),
140
+ "position_ids": torch.stack(position_id_batch),
141
+ "attention_mask": torch.stack(attention_mask_batch).unsqueeze(1),
142
+ "choice_ids": choices_batch,
143
+ "choice_indices": choice_target_ids_batch,
144
+ }
145
+
146
+ def build_inputs_for_multiple_choice(self, model_input: BatchEncoding, choices, max_length=None):
147
+ samples = [{key: value[i] for key, value in model_input.items()} for i in range(len(model_input["input_ids"]))]
148
+ samples = [self._build_input_for_multiple_choice(sample, choice) for sample, choice in
149
+ zip(samples, choices)]
150
+ inputs = self._collate(samples)
151
+ return GLMBatchEncoding(inputs)
152
+
153
+ def build_inputs_for_generation(self, model_input: BatchEncoding, max_gen_length=512, targets=None, padding=False):
154
+ mask_ids = self.mask_token_ids
155
+ input_ids = model_input.input_ids
156
+ batch_size, seq_length = input_ids.shape[:2]
157
+ position_id, block_position_id = list(range(seq_length)), [0 for _ in range(seq_length)]
158
+ position_ids, block_position_ids = [], []
159
+ labels = None
160
+ if targets is not None:
161
+ is_batched = isinstance(targets, (list, tuple))
162
+ targets = self(targets, add_special_tokens=False, padding=False).input_ids
163
+ if not is_batched:
164
+ targets = [targets]
165
+ assert len(targets) == len(input_ids)
166
+ targets = [(target + [self.eop_token_id])[:max_gen_length] for target in targets]
167
+ if not padding:
168
+ max_gen_length = max(map(len, targets))
169
+ targets = [[self.sop_token_id] + target for target in targets]
170
+ labels = [target[1:] for target in targets]
171
+ targets = [target + [self.pad_token_id] * (max_gen_length + 1 - len(target)) for target in targets]
172
+ labels = [label + [-100] * (max_gen_length - len(label)) for label in labels]
173
+ targets = torch.tensor(targets, dtype=input_ids.dtype, device=input_ids.device)
174
+ labels = torch.tensor(labels, dtype=input_ids.dtype, device=input_ids.device)
175
+ labels = torch.cat((input_ids.new_full((batch_size, seq_length), -100), labels), dim=1)
176
+ for i in range(batch_size):
177
+ mask_positions = []
178
+ for mask_id in mask_ids:
179
+ mask_positions += (input_ids[i] == mask_id).nonzero(as_tuple=True)[0].tolist()
180
+ if not mask_positions:
181
+ raise ValueError("Cannot find mask token in the input")
182
+ mask_positions.sort()
183
+ mask_pos = mask_positions[0]
184
+ position_ids.append(position_id + [mask_pos] * max_gen_length)
185
+ block_position_ids.append(block_position_id + list(range(1, max_gen_length + 1)))
186
+ position_ids = torch.tensor(position_ids, dtype=input_ids.dtype, device=input_ids.device)
187
+ block_position_ids = torch.tensor(block_position_ids, dtype=input_ids.dtype, device=input_ids.device)
188
+ position_ids = torch.stack((position_ids, block_position_ids), dim=1)
189
+ attention_mask = model_input.attention_mask
190
+ attention_mask = attention_mask.unsqueeze(1).expand(-1, seq_length + max_gen_length, -1)
191
+ generation_attention_mask = torch.cat([attention_mask.new_zeros((seq_length, max_gen_length)),
192
+ torch.tril(attention_mask.new_ones((max_gen_length, max_gen_length)))],
193
+ dim=0).unsqueeze(0).expand(batch_size, -1, -1)
194
+ attention_mask = torch.cat((attention_mask, generation_attention_mask), dim=2)
195
+ attention_mask = attention_mask.unsqueeze(1)
196
+ if targets is None:
197
+ input_ids = torch.cat((input_ids, input_ids.new_full((batch_size, 1), self.sop_token_id)), dim=-1)
198
+ else:
199
+ input_ids = torch.cat((input_ids, targets[:, :-1]), dim=1)
200
+ batch = {"input_ids": input_ids, "position_ids": position_ids}
201
+ if labels is None:
202
+ batch["generation_attention_mask"] = attention_mask
203
+ else:
204
+ batch["attention_mask"] = attention_mask
205
+ batch["labels"] = labels
206
+ return BatchEncoding(batch)
207
+
208
+
209
+ class GLMRobertaTokenizer(RobertaTokenizer, GLMTokenizerMixin):
210
+ model_input_names = ["input_ids", "position_ids", "attention_mask"]
211
+ truncation_side: str = "left"
212
+
213
+ @property
214
+ def gmask_token_id(self) -> int:
215
+ raise NotImplementedError("The model doesn't support gMASK")
216
+
217
+ @property
218
+ def smask_token_id(self) -> int:
219
+ raise NotImplementedError("The model doesn't support sMASK")
220
+
221
+ @property
222
+ def mask_token_ids(self):
223
+ return [self.mask_token_id]
224
+
225
+
226
+ class GLMChineseTokenizer(PreTrainedTokenizer, GLMTokenizerMixin):
227
+ vocab_files_names = {"vocab_file": "cog-pretrain.model"}
228
+ truncation_side: str = "left"
229
+
230
+ def __init__(self, vocab_file, **kwargs):
231
+ super().__init__(**kwargs)
232
+ self.vocab_file = vocab_file
233
+ self.sp_model = spm.SentencePieceProcessor()
234
+ self.sp_model.Load(vocab_file)
235
+
236
+ @property
237
+ def vocab_size(self):
238
+ return len(self.sp_model)
239
+
240
+ def get_vocab(self):
241
+ vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
242
+ vocab.update(self.added_tokens_encoder)
243
+ return vocab
244
+
245
+ def _tokenize(self, text, **kwargs):
246
+ return self.sp_model.encode(text, out_type=str)
247
+
248
+ def _convert_token_to_id(self, token):
249
+ """Converts a token (str) in an id using the vocab."""
250
+ return self.sp_model.PieceToId(token)
251
+
252
+ def _convert_id_to_token(self, index):
253
+ """Converts an index (integer) in a token (str) using the vocab."""
254
+ return self.sp_model.IdToPiece(index)
255
+
256
+ def convert_tokens_to_string(self, tokens):
257
+ return self.sp_model.decode(tokens)
258
+
259
+ def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
260
+ if not os.path.isdir(save_directory):
261
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
262
+ return
263
+ out_vocab_file = os.path.join(
264
+ save_directory, (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
265
+ )
266
+
267
+ if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
268
+ copyfile(self.vocab_file, out_vocab_file)
269
+ elif not os.path.isfile(self.vocab_file):
270
+ with open(out_vocab_file, "wb") as fi:
271
+ content_spiece_model = self.sp_model.serialized_model_proto()
272
+ fi.write(content_spiece_model)
273
+
274
+ return (out_vocab_file,)
275
+
276
+ def build_inputs_with_special_tokens(
277
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
278
+ ) -> List[int]:
279
+ """
280
+ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
281
+ adding special tokens. A BERT sequence has the following format:
282
+
283
+ - single sequence: ``[CLS] X [SEP]``
284
+ - pair of sequences: ``[CLS] A [SEP] B [SEP]``
285
+
286
+ Args:
287
+ token_ids_0 (:obj:`List[int]`):
288
+ List of IDs to which the special tokens will be added.
289
+ token_ids_1 (:obj:`List[int]`, `optional`):
290
+ Optional second list of IDs for sequence pairs.
291
+
292
+ Returns:
293
+ :obj:`List[int]`: List of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.
294
+ """
295
+ assert token_ids_1 is None
296
+ cls = [self.cls_token_id]
297
+ eos = [self.eos_token_id]
298
+ return cls + token_ids_0 + eos
299
+
300
+
301
+ class GLMGPT2Tokenizer(GPT2Tokenizer, GLMTokenizerMixin):
302
+ model_input_names = ["input_ids", "position_ids", "attention_mask"]
303
+ truncation_side: str = "left"
304
+
305
+ def build_inputs_with_special_tokens(
306
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
307
+ ) -> List[int]:
308
+ """
309
+ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
310
+ adding special tokens. A BERT sequence has the following format:
311
+
312
+ - single sequence: ``[CLS] X [SEP]``
313
+ - pair of sequences: ``[CLS] A [SEP] B [SEP]``
314
+
315
+ Args:
316
+ token_ids_0 (:obj:`List[int]`):
317
+ List of IDs to which the special tokens will be added.
318
+ token_ids_1 (:obj:`List[int]`, `optional`):
319
+ Optional second list of IDs for sequence pairs.
320
+
321
+ Returns:
322
+ :obj:`List[int]`: List of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.
323
+ """
324
+ assert token_ids_1 is None
325
+ cls = [self.cls_token_id]
326
+ eos = [self.eos_token_id]
327
+ return cls + token_ids_0 + eos
328
+
329
+
330
+ class GLMBertTokenizer(BertTokenizer, GLMTokenizerMixin):
331
+ model_input_names = ["input_ids", "position_ids", "attention_mask"]
332
+ truncation_side: str = "left"
333
+
334
+ @property
335
+ def gmask_token_id(self) -> int:
336
+ raise NotImplementedError("The model doesn't support gMASK")
337
+
338
+ @property
339
+ def smask_token_id(self) -> int:
340
+ raise NotImplementedError("The model doesn't support sMASK")
341
+
342
+ @property
343
+ def mask_token_ids(self):
344
+ return [self.mask_token_id]
345
+
346
+
347
+ class GLMTokenizer:
348
+ @classmethod
349
+ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
350
+ tokenizer_config = get_tokenizer_config(pretrained_model_name_or_path, **kwargs)
351
+ config_tokenizer_class = tokenizer_config.get("tokenizer_class")
352
+ if config_tokenizer_class == "GLMRobertaTokenizer":
353
+ tokenizer_class = GLMRobertaTokenizer
354
+ elif config_tokenizer_class == "GLMChineseTokenizer":
355
+ tokenizer_class = GLMChineseTokenizer
356
+ elif config_tokenizer_class == "GLMGPT2Tokenizer":
357
+ tokenizer_class = GLMGPT2Tokenizer
358
+ elif config_tokenizer_class == "GLMBertTokenizer":
359
+ tokenizer_class = GLMBertTokenizer
360
+ else:
361
+ raise NotImplementedError("Not implemented tokenizer type:", config_tokenizer_class)
362
+ return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
tokenizer_config.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name_or_path": "THUDM/glm-large-chinese",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|endoftext|>",
5
+ "cls_token": "[CLS]",
6
+ "mask_token": "[MASK]",
7
+ "unk_token": "[UNK]",
8
+ "additional_special_tokens": ["<|startofpiece|>", "<|endofpiece|>", "[gMASK]", "[sMASK]"],
9
+ "add_prefix_space": false,
10
+ "tokenizer_class": "GLMChineseTokenizer",
11
+ "use_fast": false,
12
+ "auto_map": {
13
+ "AutoTokenizer": [
14
+ "tokenization_glm.GLMChineseTokenizer",
15
+ null
16
+ ]
17
+ }
18
+ }