zxdu20 commited on
Commit
7a1cab8
1 Parent(s): 46cd547

init commit

Browse files
README.md ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - zh
4
+ tags:
5
+ - glm
6
+ ---
7
+ GLM is a General Language Model pretrained with an autoregressive blank-filling objective and can be finetuned on various natural language understanding and generation tasks.
8
+
9
+ Please refer to our paper for a detailed description of GLM:
10
+
11
+ [GLM: General Language Model Pretraining with Autoregressive Blank Infilling](https://arxiv.org/abs/2103.10360) (ACL 2022)
12
+
13
+ Zhengxiao Du*, Yujie Qian*, Xiao Liu, Ming Ding, Jiezhong Qiu, Zhilin Yang, Jie Tang (*: equal contribution)
14
+
15
+ Find more examples in our [Github repo](https://github.com/THUDM/GLM).
16
+
17
+ ## Model description
18
+ `glm-10b-chinese` is pretrained on the [WuDaoCorpora](https://www.sciencedirect.com/science/article/pii/S2666651021000152) dataset. It has 48 transformer layers, with hidden size 4096 and 64 attention heads in each layer. The model is pretrained with autoregressive blank filling objectives designed for natural language understanding, seq2seq, and language modeling.
19
+
20
+ ## How to use
21
+ ```python
22
+ from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
23
+ tokenizer = AutoTokenizer.from_pretrained("BAAI/glm-10b-chinese", trust_remote_code=True)
24
+ model = AutoModelForSeq2SeqLM.from_pretrained("BAAI/glm-10b-chinese", trust_remote_code=True)
25
+ model = model.half().cuda()
26
+
27
+ inputs = tokenizer("凯旋门位于意大利米兰市古城堡旁。1807年为纪念[MASK]而建,门高25米,顶上矗立两武士青铜古兵车铸像。", return_tensors="pt")
28
+ inputs = tokenizer.build_inputs_for_generation(inputs, max_gen_length=512)
29
+ inputs = {key: value.cuda() for key, value in inputs.items()}
30
+ outputs = model.generate(**inputs, max_length=512, eos_token_id=tokenizer.eop_token_id)
31
+ print(tokenizer.decode(outputs[0].tolist()))
32
+ ```
33
+ We use three different mask tokens for different tasks: `[MASK]` for short blank filling, `[sMASK]` for sentence filling, and `[gMASK]` for left to right generation. You can find examples about different masks from [here](https://github.com/THUDM/GLM#left-to-right-generation--blank-filling-interactive).
34
+
35
+ ## Citation
36
+ Please cite our paper if you find this code useful for your research:
37
+ ```
38
+ @article{DBLP:conf/acl/DuQLDQY022,
39
+ author = {Zhengxiao Du and
40
+ Yujie Qian and
41
+ Xiao Liu and
42
+ Ming Ding and
43
+ Jiezhong Qiu and
44
+ Zhilin Yang and
45
+ Jie Tang},
46
+ title = {{GLM:} General Language Model Pretraining with Autoregressive Blank Infilling},
47
+ booktitle = {Proceedings of the 60th Annual Meeting of the Association for Computational
48
+ Linguistics (Volume 1: Long Papers), {ACL} 2022, Dublin, Ireland,
49
+ May 22-27, 2022},
50
+ pages = {320--335},
51
+ publisher = {Association for Computational Linguistics},
52
+ year = {2022},
53
+ }
54
+ ```
added_tokens.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<|endoftext|>": 50000,
3
+ "[SEP]": 50001,
4
+ "[CLS]": 50002,
5
+ "[MASK]": 50003,
6
+ "[UNUSED1]": 50004,
7
+ "[UNUSED2]": 50005,
8
+ "<|startofpiece|>": 50006,
9
+ "<|endofpiece|>": 50007,
10
+ "[sMASK]": 50008,
11
+ "[gMASK]": 50009
12
+ }
cog-pretrain.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ea6f4164152bc58d23e24e48f7bf4187aad72a32e97ec4b3acc832fe183cbc2
3
+ size 1021864
config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "THUDM/glm-10b-chinese",
3
+ "architectures": [
4
+ "GLMModel"
5
+ ],
6
+ "attention_dropout_prob": 0.1,
7
+ "attention_scale": 1.0,
8
+ "auto_map": {
9
+ "AutoConfig": "configuration_glm.GLMConfig",
10
+ "AutoModel": "modeling_glm.GLMModel",
11
+ "AutoModelForSequenceClassification": "modeling_glm.GLMForSequenceClassification",
12
+ "AutoModelForSeq2SeqLM": "modeling_glm.GLMForConditionalGeneration"
13
+ },
14
+ "block_position_encoding": true,
15
+ "checkpoint_activations": false,
16
+ "checkpoint_num_layers": 1,
17
+ "embedding_dropout_prob": 0.1,
18
+ "hidden_size": 4096,
19
+ "initializer_range": 0.02,
20
+ "max_sequence_length": 1024,
21
+ "model_type": "glm",
22
+ "num_attention_heads": 64,
23
+ "num_layers": 48,
24
+ "output_dropout_prob": 0.1,
25
+ "output_predict": true,
26
+ "parallel_output": true,
27
+ "pool_token": "cls",
28
+ "relative_encoding": false,
29
+ "spell_func": "lstm",
30
+ "spell_length": null,
31
+ "torch_dtype": "float32",
32
+ "transformers_version": "4.23.1",
33
+ "vocab_size": 50048,
34
+ "bad_words_id": [50009]
35
+ }
configuration_glm.py ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2022 shunxing1234 and The HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """ GLM model configuration """
16
+
17
+ from transformers.configuration_utils import PretrainedConfig
18
+ from transformers.utils import logging
19
+
20
+ logger = logging.get_logger(__name__)
21
+
22
+ GLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {
23
+ "shunxing1234/GLM": "https://huggingface.co/shunxing1234/GLM/resolve/main/config.json",
24
+ # See all GLM models at https://huggingface.co/models?filter=glm
25
+ }
26
+
27
+
28
+ class GLMConfig(PretrainedConfig):
29
+ r"""
30
+ This is the configuration class to store the configuration of a [`~GLMModel`].
31
+ It is used to instantiate an GLM model according to the specified arguments, defining the model
32
+ architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of
33
+ the GLM [shunxing1234/GLM-base-cased](https://huggingface.co/shunxing1234/GLM-base-cased) architecture.
34
+
35
+ Configuration objects inherit from [`PretrainedConfig`] and can be used
36
+ to control the model outputs. Read the documentation from [`PretrainedConfig`]
37
+ for more information.
38
+
39
+
40
+ Args:
41
+ vocab_size (`int`, *optional*, defaults to 30522):
42
+ Vocabulary size of the GLM model. Defines the number of different tokens that can be represented by the
43
+ `inputs_ids` passed when calling [`~GLMModel`] or
44
+ [`~TFGLMModel`].
45
+ hidden_size (`int`, *optional*, defaults to 768):
46
+ Dimension of the encoder layers and the pooler layer.
47
+ num_hidden_layers (`int`, *optional*, defaults to 12):
48
+ Number of hidden layers in the Transformer encoder.
49
+ num_attention_heads (`int`, *optional*, defaults to 12):
50
+ Number of attention heads for each attention layer in the Transformer encoder.
51
+ intermediate_size (`int`, *optional*, defaults to 3072):
52
+ Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
53
+ hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
54
+ The non-linear activation function (function or string) in the encoder and pooler.
55
+ If string, `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
56
+ hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
57
+ The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
58
+ attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
59
+ The dropout ratio for the attention probabilities.
60
+ max_position_embeddings (`int`, *optional*, defaults to 512):
61
+ The maximum sequence length that this model might ever be used with.
62
+ Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
63
+ type_vocab_size (`int`, *optional*, defaults to 2):
64
+ The vocabulary size of the `token_type_ids` passed when calling [`~GLMModel`] or
65
+ [`~TFGLMModel`].
66
+ initializer_range (`float`, *optional*, defaults to 0.02):
67
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
68
+ layer_norm_eps (`float`, *optional*, defaults to 1e-12):
69
+ The epsilon used by the layer normalization layers.
70
+ use_cache (`bool`, *optional*, defaults to `True`):
71
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
72
+ relevant if `config.is_decoder=True`.
73
+ Example:
74
+
75
+ ```python
76
+ >>> from transformers import GLMModel, GLMConfig
77
+
78
+ >>> # Initializing a GLM shunxing1234/GLM-base-cased style configuration
79
+ >>> configuration = GLMConfig()
80
+
81
+ >>> # Initializing a model from the shunxing1234/GLM-base-cased style configuration
82
+ >>> model = GLMModel(configuration)
83
+
84
+ >>> # Accessing the model configuration
85
+ >>> configuration = model.config
86
+ ```
87
+ """
88
+ model_type = "glm"
89
+ attribute_map = {
90
+ "num_hidden_layers": "num_layers"
91
+ }
92
+
93
+ def __init__(
94
+ self,
95
+ num_layers=24,
96
+ vocab_size=30592,
97
+ hidden_size=1024,
98
+ num_attention_heads=16,
99
+ embedding_dropout_prob=0.1,
100
+ attention_dropout_prob=0.1,
101
+ output_dropout_prob=0.1,
102
+ max_sequence_length=512,
103
+ checkpoint_activations=False,
104
+ checkpoint_num_layers=1,
105
+ parallel_output=True,
106
+ relative_encoding=False,
107
+ block_position_encoding=True,
108
+ output_predict=False,
109
+ spell_length=None,
110
+ spell_func="lstm",
111
+ attention_scale=1.0,
112
+ initializer_range=0.02,
113
+ pool_token="cls",
114
+ **kwargs
115
+ ):
116
+ self.num_layers = num_layers
117
+ self.vocab_size = vocab_size
118
+ self.hidden_size = hidden_size
119
+ self.num_attention_heads = num_attention_heads
120
+ self.embedding_dropout_prob = embedding_dropout_prob
121
+ self.attention_dropout_prob = attention_dropout_prob
122
+ self.output_dropout_prob = output_dropout_prob
123
+ self.max_sequence_length = max_sequence_length
124
+ self.checkpoint_activations = checkpoint_activations
125
+ self.checkpoint_num_layers = checkpoint_num_layers
126
+ self.parallel_output = parallel_output
127
+ self.relative_encoding = relative_encoding
128
+ self.block_position_encoding = block_position_encoding
129
+ self.output_predict = output_predict
130
+ self.spell_length = spell_length
131
+ self.spell_func = spell_func
132
+ self.attention_scale = attention_scale
133
+ self.initializer_range = initializer_range
134
+ self.pool_token = pool_token
135
+
136
+ super().__init__(**kwargs)
modeling_glm.py ADDED
@@ -0,0 +1,919 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2022 shunxing1234 The HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """ PyTorch GLM model. """
16
+
17
+ import math
18
+
19
+ import torch
20
+ import torch.utils.checkpoint
21
+ import torch.nn.functional as F
22
+ from torch.nn import init, LayerNorm, Linear, CrossEntropyLoss
23
+
24
+ from transformers.activations import gelu
25
+ from transformers.utils import (
26
+ add_code_sample_docstrings,
27
+ add_start_docstrings,
28
+ add_start_docstrings_to_model_forward,
29
+ )
30
+ from transformers.modeling_outputs import (
31
+ BaseModelOutputWithPastAndCrossAttentions,
32
+ SequenceClassifierOutput,
33
+ ModelOutput
34
+ )
35
+ from transformers.modeling_utils import (
36
+ PreTrainedModel,
37
+ )
38
+ from transformers.utils import logging
39
+ from .configuration_glm import GLMConfig
40
+ from torch.nn.parameter import Parameter
41
+
42
+ _CHECKPOINT_FOR_DOC = "shunxing1234/GLM"
43
+ _CONFIG_FOR_DOC = "GLMConfig"
44
+ _TOKENIZER_FOR_DOC = "GLMTokenizer"
45
+
46
+ GLM_PRETRAINED_MODEL_ARCHIVE_LIST = [
47
+ "shunxing1234/GLM",
48
+ # See all GLM models at https://huggingface.co/models?filter=glm
49
+ ]
50
+
51
+
52
+ def unscaled_init_method(sigma):
53
+ """Init method based on N(0, sigma)."""
54
+
55
+ def init_(tensor):
56
+ return torch.nn.init.normal_(tensor, mean=0.0, std=sigma)
57
+
58
+ return init_
59
+
60
+
61
+ def scaled_init_method(mean, std, num_layers):
62
+ """Init method based on N(0, sigma/sqrt(2*num_layers)."""
63
+ std = std / math.sqrt(2.0 * num_layers)
64
+
65
+ def init_(tensor):
66
+ return torch.nn.init.normal_(tensor, mean=mean, std=std)
67
+
68
+ return init_
69
+
70
+
71
+ def ensure_divisibility(numerator, denominator):
72
+ """Ensure that numerator is divisible by the denominator."""
73
+ assert numerator % denominator == 0, '{} is not divisible by {}'.format(
74
+ numerator, denominator)
75
+
76
+
77
+ def divide(numerator, denominator):
78
+ """Ensure that numerator is divisible by the denominator and return
79
+ the division value."""
80
+ ensure_divisibility(numerator, denominator)
81
+ return numerator // denominator
82
+
83
+
84
+ def split_tensor_along_last_dim(tensor, num_partitions,
85
+ contiguous_split_chunks=False):
86
+ """Split a tensor along its last dimension.
87
+ Arguments:
88
+ tensor: input tensor.
89
+ num_partitions: number of partitions to split the tensor
90
+ contiguous_split_chunks: If True, make each chunk contiguous
91
+ in memory.
92
+ """
93
+ # Get the size and dimension.
94
+ last_dim = tensor.dim() - 1
95
+ last_dim_size = divide(tensor.size()[last_dim], num_partitions)
96
+ # Split.
97
+ tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
98
+ # Note: torch.split does not create contiguous tensors by default.
99
+ if contiguous_split_chunks:
100
+ return tuple(chunk.contiguous() for chunk in tensor_list)
101
+
102
+ return tensor_list
103
+
104
+
105
+ class MLP(torch.nn.Module):
106
+ """MLP for GPT2.
107
+
108
+ MLP will take the input with h hidden state, project it to 4*h
109
+ hidden dimension, perform gelu transformation, and project the
110
+ state back into h hidden dimension. At the end, dropout is also
111
+ applied.
112
+
113
+ Arguments:
114
+ hidden_size: The hidden size of the self attention.
115
+ output_dropout_prob: dropout probability for the outputs
116
+ after self attention and final output.
117
+ init_method: initialization method used for the weights. Note
118
+ that all biases are initialized to zero and
119
+ layernorm weight are initialized to one.
120
+ output_layer_init_method: output layer initialization. If None,
121
+ use `init_method`.
122
+ """
123
+
124
+ def __init__(self, hidden_size, output_dropout_prob, init_method,
125
+ output_layer_init_method=None):
126
+ super(MLP, self).__init__()
127
+ # Set output layer initialization if not provided.
128
+ if output_layer_init_method is None:
129
+ output_layer_init_method = init_method
130
+ # Project to 4h.
131
+ self.dense_h_to_4h = Linear(hidden_size, 4 * hidden_size)
132
+
133
+ # Project back to h.
134
+ self.dense_4h_to_h = Linear(
135
+ 4 * hidden_size,
136
+ hidden_size)
137
+
138
+ self.dropout = torch.nn.Dropout(output_dropout_prob)
139
+
140
+ def forward(self, hidden_states):
141
+ # [b, s, 4hp]
142
+ intermediate_parallel = self.dense_h_to_4h(hidden_states)
143
+ intermediate_parallel = gelu(intermediate_parallel)
144
+
145
+ # [b, s, h]
146
+ output = self.dense_4h_to_h(intermediate_parallel)
147
+ output = self.dropout(output)
148
+ return output
149
+
150
+
151
+ class VocabEmbedding(torch.nn.Module):
152
+ """Embedding parallelized in the vocabulary dimension.
153
+
154
+ This is mainly adapted from torch.nn.Embedding and all the default
155
+ values are kept.
156
+ Arguments:
157
+ num_embeddings: vocabulary size.
158
+ embedding_dim: size of hidden state.
159
+ init_method: method to initialize weights.
160
+ """
161
+
162
+ def __init__(self, config):
163
+ super(VocabEmbedding, self).__init__()
164
+ # Keep the input dimensions.
165
+ self.num_embeddings = config.vocab_size
166
+ self.embedding_dim = config.hidden_size
167
+ # Set the detauls for compatibility.
168
+ self.padding_idx = None
169
+ self.max_norm = None
170
+ self.norm_type = 2.
171
+ self.scale_grad_by_freq = False
172
+ self.sparse = False
173
+ self._weight = None
174
+
175
+ self.vocab_start_index = 0
176
+ self.vocab_end_index = self.num_embeddings
177
+
178
+ # Allocate weights.
179
+ self.weight = Parameter(torch.Tensor(self.num_embeddings,
180
+ self.embedding_dim))
181
+ # And initialize.
182
+ init.xavier_normal_(self.weight)
183
+
184
+ def forward(self, input_):
185
+ # Get the embeddings.
186
+ output = F.embedding(input_, self.weight,
187
+ self.padding_idx, self.max_norm,
188
+ self.norm_type, self.scale_grad_by_freq,
189
+ self.sparse)
190
+ return output
191
+
192
+
193
+ class PositionalEmbedding(torch.nn.Module):
194
+
195
+ def __init__(self, hidden_size):
196
+ super(PositionalEmbedding, self).__init__()
197
+
198
+ self.hidden_size = hidden_size
199
+
200
+ inv_freq = 1 / (10000 ** (torch.arange(0.0, hidden_size, 2.0) / hidden_size))
201
+ self.register_buffer('inv_freq', inv_freq)
202
+
203
+ def forward(self, pos_seq, bsz=None):
204
+ sinusoid_inp = torch.ger(pos_seq, self.inv_freq)
205
+ pos_emb = torch.cat([sinusoid_inp.sin(), sinusoid_inp.cos()], dim=-1)
206
+
207
+ if bsz is not None:
208
+ return pos_emb[None, :, :].expand(bsz, -1, -1)
209
+ else:
210
+ return pos_emb[None, :, :]
211
+
212
+
213
+ class SelfAttention(torch.nn.Module):
214
+ """self-attention layer for GLM.
215
+
216
+ Self-attention layer takes input with size [b, s, h] where b is
217
+ the batch size, s is the sequence lenght, and h is the hidden size
218
+ and creates output of the same size.
219
+ Arguments:
220
+ hidden_size: total hidden size of the layer (h).
221
+ num_attention_heads: number of attention heads (n). Note that we
222
+ require n to be divisible by number of GPUs
223
+ used to parallelize the model. Also, we
224
+ require hidden size to be divisible by n.
225
+ attention_dropout_prob: dropout probability for the attention scores.
226
+ init_method: weight initialization.
227
+ output_layer_init_method: output layer initialization. If None, use
228
+ `init_method`.
229
+ We use the following notation:
230
+ h: hidden_size
231
+ n: num_attention_heads
232
+ p: number of partitions
233
+ np: n/p
234
+ hp: h/p
235
+ hn: h/n
236
+ b: batch size
237
+ s: sequence length
238
+ """
239
+
240
+ def __init__(self, hidden_size, num_attention_heads,
241
+ attention_dropout_prob, output_dropout_prob,
242
+ init_method, output_layer_init_method=None,
243
+ attention_scale=1.0):
244
+ super(SelfAttention, self).__init__()
245
+ # Set output layer initialization if not provided.
246
+ if output_layer_init_method is None:
247
+ output_layer_init_method = init_method
248
+ # Per attention head and per partition values.
249
+ self.hidden_size = hidden_size
250
+ self.hidden_size_per_attention_head = divide(hidden_size,
251
+ num_attention_heads)
252
+
253
+ self.num_attention_heads = num_attention_heads
254
+ self.attention_scale = attention_scale
255
+ # Strided linear layer.
256
+ self.query_key_value = Linear(hidden_size, 3 * hidden_size)
257
+
258
+ # Dropout. Note that for a single iteration, this layer will generate
259
+ # different outputs on different number of parallel partitions but
260
+ # on average it should not be partition dependent.
261
+ self.attention_dropout = torch.nn.Dropout(attention_dropout_prob)
262
+
263
+ # Output.
264
+ self.dense = Linear(hidden_size,
265
+ hidden_size)
266
+ self.output_dropout = torch.nn.Dropout(output_dropout_prob)
267
+
268
+ def _transpose_for_scores(self, tensor):
269
+ """Transpose a 3D tensor [b, s, np*hn] into a 4D tensor with
270
+ size [b, np, s, hn].
271
+ """
272
+ new_tensor_shape = tensor.size()[:-1] + \
273
+ (self.num_attention_heads,
274
+ self.hidden_size_per_attention_head)
275
+ tensor = tensor.view(*new_tensor_shape)
276
+ return tensor.permute(0, 2, 1, 3)
277
+
278
+ def forward(self, hidden_states, ltor_mask, mem=None):
279
+ # hidden_states: [b, s, h]
280
+ # ltor_mask: [b,1,s,s]
281
+
282
+ # Attention heads. [b, s, hp]
283
+ query_length = hidden_states.size(1)
284
+ # self attention
285
+ if mem is None:
286
+ mixed_x_layer = self.query_key_value(hidden_states)
287
+ (mixed_query_layer,
288
+ mixed_key_layer,
289
+ mixed_value_layer) = split_tensor_along_last_dim(mixed_x_layer, 3)
290
+ else:
291
+ cat = torch.cat((mem, hidden_states), 1)
292
+ mixed_x_layer = self.query_key_value(cat)
293
+ (mixed_query_layer,
294
+ mixed_key_layer,
295
+ mixed_value_layer) = split_tensor_along_last_dim(mixed_x_layer, 3)
296
+ mixed_query_layer = mixed_query_layer[:, -query_length:]
297
+
298
+ # Reshape and transpose [b, np, s, hn]
299
+ query_layer = self._transpose_for_scores(mixed_query_layer)
300
+ key_layer = self._transpose_for_scores(mixed_key_layer)
301
+ value_layer = self._transpose_for_scores(mixed_value_layer)
302
+
303
+ if self.attention_scale > 1.0:
304
+ # Raw attention scores. [b, np, s, s]
305
+ attention_scores = torch.matmul(query_layer / math.sqrt(self.attention_scale),
306
+ key_layer.transpose(-1, -2) / math.sqrt(
307
+ self.hidden_size_per_attention_head * self.attention_scale))
308
+ else:
309
+ attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2) / math.sqrt(
310
+ self.hidden_size_per_attention_head))
311
+
312
+ # Apply the left to right attention mask.
313
+ ltor_mask = ltor_mask.type_as(attention_scores)
314
+ attention_scores = torch.mul(attention_scores, ltor_mask)
315
+ if self.attention_scale > 1.0:
316
+ max_attention_scores = attention_scores.max(dim=-1, keepdim=True)[0]
317
+ attention_scores -= max_attention_scores
318
+ attention_scores *= self.attention_scale
319
+
320
+ attention_scores = attention_scores + (-65504.0) * (1.0 - ltor_mask)
321
+ # Attention probabilities. [b, np, s, s]
322
+ attention_probs = torch.nn.Softmax(dim=-1)(attention_scores)
323
+ # This is actually dropping out entire tokens to attend to, which might
324
+ # seem a bit unusual, but is taken from the original Transformer paper.
325
+ # with get_cuda_rng_tracker().fork():
326
+ attention_probs = self.attention_dropout(attention_probs)
327
+
328
+ # Context layer.
329
+ # [b, np, s, hn]
330
+ context_layer = torch.matmul(attention_probs, value_layer)
331
+ # [b, s, np, hn]
332
+ context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
333
+ new_context_layer_shape = context_layer.size()[:-2] + \
334
+ (self.hidden_size,)
335
+ # [b, s, hp]
336
+ context_layer = context_layer.view(*new_context_layer_shape)
337
+
338
+ # Output. [b, s, h]
339
+ output = self.dense(context_layer)
340
+ output = self.output_dropout(output)
341
+
342
+ return output
343
+
344
+
345
+ class GLMBlock(torch.nn.Module):
346
+ """A single layer transformer for GLM.
347
+
348
+ We use the following notation:
349
+ h: hidden size
350
+ n: number of attention heads
351
+ b: batch size
352
+ s: sequence length
353
+ Transformore layer takes input with size [b, s, h] and returns an
354
+ output of the same size.
355
+
356
+ Arguments:
357
+ hidden_size: The hidden size of the self attention.
358
+ num_attention_heads: number of attention head in the self
359
+ attention.
360
+ attention_dropout_prob: dropout probability of the attention
361
+ score in self attention.
362
+ output_dropout_prob: dropout probability for the outputs
363
+ after self attention and final output.
364
+ layernorm_epsilon: epsilon used in layernorm to avoid
365
+ division by zero.
366
+ init_method: initialization method used for the weights. Note
367
+ that all biases are initialized to zero and
368
+ layernorm weight are initialized to one.
369
+ output_layer_init_method: output layers (attention output and
370
+ mlp output) initialization. If None,
371
+ use `init_method`.
372
+ """
373
+
374
+ def __init__(self,
375
+ hidden_size,
376
+ num_attention_heads,
377
+ attention_dropout_prob,
378
+ output_dropout_prob,
379
+ layernorm_epsilon,
380
+ init_method,
381
+ output_layer_init_method=None,
382
+ attention_scale=1.0):
383
+ super(GLMBlock, self).__init__()
384
+ # Set output layer initialization if not provided.
385
+ if output_layer_init_method is None:
386
+ output_layer_init_method = init_method
387
+
388
+ # Layernorm on the input data.
389
+ self.input_layernorm = LayerNorm(hidden_size, eps=layernorm_epsilon)
390
+
391
+ # Self attention.
392
+ self.attention = SelfAttention(
393
+ hidden_size,
394
+ num_attention_heads,
395
+ attention_dropout_prob,
396
+ output_dropout_prob,
397
+ init_method,
398
+ output_layer_init_method=output_layer_init_method,
399
+ attention_scale=attention_scale)
400
+
401
+ # Layernorm on the input data.
402
+ self.post_attention_layernorm = LayerNorm(hidden_size,
403
+ eps=layernorm_epsilon)
404
+
405
+ # MLP
406
+ self.mlp = MLP(
407
+ hidden_size,
408
+ output_dropout_prob,
409
+ init_method,
410
+ output_layer_init_method=output_layer_init_method)
411
+
412
+ def forward(self, hidden_states, ltor_mask, mem=None):
413
+ # hidden_states: [b, s, h]
414
+ # ltor_mask: [b,1, s,s]
415
+
416
+ # Layer norm at the begining of the transformer layer.
417
+ layernorm_output = self.input_layernorm(hidden_states)
418
+ mem = self.input_layernorm(mem) if mem is not None else None
419
+ # Self attention.
420
+ attention_output = self.attention(layernorm_output, ltor_mask, mem)
421
+ # Residual connection.
422
+ layernorm_input = hidden_states + attention_output
423
+ # Layer norm post the self attention.
424
+ layernorm_output = self.post_attention_layernorm(layernorm_input)
425
+ # MLP.
426
+ mlp_output = self.mlp(layernorm_output)
427
+ # Second residual connection.
428
+ output = layernorm_input + mlp_output
429
+
430
+ return output
431
+
432
+
433
+ class GLMStack(torch.nn.Module):
434
+ """GLM transformer.
435
+
436
+ This module takes input from embedding layer and it's output can
437
+ be used directly by a logit layer. It consists of L (num-layers)
438
+ blocks of:
439
+ layer norm
440
+ self attention
441
+ residual connection
442
+ layer norm
443
+ mlp
444
+ residual connection
445
+ followed by a final layer norm.
446
+
447
+ Arguments:
448
+ num_layers: Number of transformer layers.
449
+ hidden_size: The hidden size of the self attention.
450
+ num_attention_heads: number of attention head in the self
451
+ attention.
452
+ attention_dropout_prob: dropout probability of the attention
453
+ score in self attention.
454
+ output_dropout_prob: dropout probability for the outputs
455
+ after self attention and final output.
456
+ checkpoint_activations: if True, checkpoint activations.
457
+ checkpoint_num_layers: number of layers to checkpoint. This
458
+ is basically the chunk size in checkpoitning.
459
+ layernorm_epsilon: epsilon used in layernorm to avoid
460
+ division by zero.
461
+ init_method_std: standard deviation of the init method which has
462
+ the form N(0, std).
463
+ use_scaled_init_for_output_weights: If Ture use 1/sqrt(2*num_layers)
464
+ scaling for the output weights (
465
+ output of self attention and mlp).
466
+ """
467
+
468
+ def __init__(self,
469
+ num_layers,
470
+ hidden_size,
471
+ num_attention_heads,
472
+ max_sequence_length,
473
+ embedding_dropout_prob,
474
+ attention_dropout_prob,
475
+ output_dropout_prob,
476
+ checkpoint_activations,
477
+ checkpoint_num_layers=1,
478
+ layernorm_epsilon=1.0e-5,
479
+ init_method_std=0.02,
480
+ use_scaled_init_for_output_weights=True,
481
+ block_position_encoding=False,
482
+ attention_scale=1.0,
483
+ ):
484
+ super(GLMStack, self).__init__()
485
+ self.hidden_size = hidden_size
486
+ # Store activation checkpoiting flag.
487
+ self.checkpoint_activations = checkpoint_activations
488
+ self.checkpoint_num_layers = checkpoint_num_layers
489
+
490
+ output_layer_init_method = None
491
+ if use_scaled_init_for_output_weights:
492
+ output_layer_init_method = scaled_init_method(0.0, init_method_std,
493
+ num_layers)
494
+ # Embeddings dropout
495
+ self.embedding_dropout = torch.nn.Dropout(embedding_dropout_prob)
496
+ self.block_position_encoding = block_position_encoding
497
+
498
+ # Position embedding (serial).
499
+ if block_position_encoding:
500
+ self.position_embeddings = torch.nn.Embedding(max_sequence_length + 1, hidden_size)
501
+ self.block_position_embeddings = torch.nn.Embedding(max_sequence_length + 1, hidden_size)
502
+ torch.nn.init.normal_(self.block_position_embeddings.weight, mean=0.0, std=init_method_std)
503
+ else:
504
+ self.position_embeddings = torch.nn.Embedding(max_sequence_length, hidden_size)
505
+ # Initialize the position embeddings.
506
+ torch.nn.init.normal_(self.position_embeddings.weight, mean=0.0, std=init_method_std)
507
+
508
+ def get_layer():
509
+
510
+ return GLMBlock(
511
+ hidden_size,
512
+ num_attention_heads,
513
+ attention_dropout_prob,
514
+ output_dropout_prob,
515
+ layernorm_epsilon,
516
+ unscaled_init_method(init_method_std),
517
+ output_layer_init_method=output_layer_init_method,
518
+ attention_scale=attention_scale)
519
+
520
+ # Transformer layers.
521
+ self.layers = torch.nn.ModuleList(
522
+ [get_layer() for _ in range(num_layers)])
523
+
524
+ # Final layer norm before output.
525
+ self.final_layernorm = LayerNorm(hidden_size, eps=layernorm_epsilon)
526
+
527
+ def forward(self, hidden_states, position_ids, attention_mask, memory_states=None):
528
+
529
+ batch_size, query_length = hidden_states.size()[:2]
530
+ memory_length = memory_states[0].size(1) if memory_states else 0
531
+ # attention mask is the beginning postion of B region, \in [0, query_len)
532
+ is_scalar = torch.numel(attention_mask) == 1
533
+ is_sep = is_scalar or torch.numel(attention_mask) == batch_size
534
+ if is_sep:
535
+ sep = attention_mask.item() if is_scalar else attention_mask
536
+
537
+ # conventional transformer
538
+ def build_mask_matrix(seq_length, sep, memory_length=0):
539
+ m = hidden_states.new_ones((1, seq_length, seq_length))
540
+ m = torch.tril(m)
541
+ if is_scalar:
542
+ m[0, :, :int(sep)] = 1
543
+ else:
544
+ m = m.expand(batch_size, -1, -1)
545
+ ids = torch.arange(seq_length, device=sep.device, dtype=sep.dtype).view(1, -1)
546
+ mask = ids < sep.view(-1, 1)
547
+ m = m.masked_fill(mask.unsqueeze(1).expand_as(m), 1)
548
+ if memory_length > 0:
549
+ m = m.expand(batch_size, -1, -1)
550
+ m = torch.cat((hidden_states.new_ones((batch_size, seq_length, memory_length)), m), dim=2)
551
+ m = m.unsqueeze(1)
552
+ return m
553
+
554
+ attention_mask = build_mask_matrix(query_length, sep, memory_length=memory_length)
555
+ else:
556
+ if attention_mask.dim() == 2:
557
+ attention_mask = attention_mask.unsqueeze(1).unsqueeze(1)
558
+ attention_mask = attention_mask[:, :, :, -query_length - memory_length:]
559
+
560
+ if self.block_position_encoding:
561
+ position_ids, block_position_ids = position_ids[:, 0], position_ids[:, 1]
562
+ position_embeddings = self.position_embeddings(position_ids)
563
+
564
+ hidden_states = hidden_states + position_embeddings
565
+ if self.block_position_encoding:
566
+ block_position_embeddings = self.block_position_embeddings(block_position_ids)
567
+ hidden_states = hidden_states + block_position_embeddings
568
+ hidden_states = self.embedding_dropout(hidden_states)
569
+
570
+ def check_detach(_hidden_states):
571
+ return _hidden_states.detach()
572
+
573
+ mem_layers = [check_detach(hidden_states)]
574
+
575
+ for i, layer in enumerate(self.layers):
576
+
577
+ args = [hidden_states, attention_mask]
578
+
579
+ def create_custom_forward(module):
580
+ def custom_forward(*inputs):
581
+ # None for past_key_value
582
+ return module(*inputs)
583
+
584
+ return custom_forward
585
+
586
+ mem_i = memory_states[i] if memory_states else None
587
+
588
+ if self.checkpoint_activations:
589
+ hidden_states = torch.utils.checkpoint.checkpoint(
590
+ create_custom_forward(layer),
591
+ hidden_states,
592
+ mem=mem_i,
593
+ )
594
+ else:
595
+ hidden_states = layer(*args, mem=mem_i)
596
+ mem_layers.append(check_detach(hidden_states))
597
+
598
+ # Final layer norm.
599
+ output = self.final_layernorm(hidden_states)
600
+ mem_layers = self.update_mems(mem_layers, memory_states)
601
+ return (output, mem_layers)
602
+
603
+ def update_mems(self, hiddens, mems):
604
+ memory_length = mems[0].size(1) if mems else 0
605
+ query_length = hiddens[0].size(1)
606
+ new_memory_length = memory_length + query_length
607
+
608
+ new_mems = []
609
+ # with torch.no_grad():
610
+ for i in range(len(hiddens)):
611
+ if new_memory_length <= query_length:
612
+ new_mems.append(hiddens[i][:, -new_memory_length:])
613
+ else:
614
+ new_mems.append(torch.cat((mems[i][:, -new_memory_length + query_length:], hiddens[i]), dim=1))
615
+ return new_mems
616
+
617
+
618
+ class GLMPreTrainedModel(PreTrainedModel):
619
+ """
620
+ An abstract class to handle weights initialization and
621
+ a simple interface for downloading and loading pretrained models.
622
+ """
623
+
624
+ config_class = GLMConfig
625
+ base_model_prefix = "glm"
626
+ supports_gradient_checkpointing = True
627
+ _keys_to_ignore_on_load_missing = [r"position_ids"]
628
+
629
+ def _init_weights(self, module):
630
+ """ Initialize the weights """
631
+ if isinstance(module, torch.nn.Linear):
632
+ # Slightly different from the TF version which uses truncated_normal for initialization
633
+ # cf https://github.com/pytorch/pytorch/pull/5617
634
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
635
+ if module.bias is not None:
636
+ module.bias.data.zero_()
637
+ elif isinstance(module, torch.nn.Embedding):
638
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
639
+ if module.padding_idx is not None:
640
+ module.weight.data[module.padding_idx].zero_()
641
+ elif isinstance(module, torch.nn.LayerNorm):
642
+ module.bias.data.zero_()
643
+ module.weight.data.fill_(1.0)
644
+
645
+ def _set_gradient_checkpointing(self, module, value=False):
646
+ if isinstance(module, GLMModel):
647
+ module.gradient_checkpointing = value
648
+
649
+
650
+ GLM_START_DOCSTRING = r"""
651
+ This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class.
652
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general
653
+ usage and behavior.
654
+
655
+ Parameters:
656
+ config ([`~GLMConfig`]): Model configuration class with all the parameters of the model.
657
+ Initializing with a config file does not load the weights associated with the model, only the configuration.
658
+ Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
659
+ """
660
+
661
+ GLM_INPUTS_DOCSTRING = r"""
662
+ Args:
663
+ input_ids (`torch.LongTensor` of shape `({0})`):
664
+ Indices of input sequence tokens in the vocabulary.
665
+
666
+ Indices can be obtained using [`GLMTokenizer`].
667
+ See [`PreTrainedTokenizer.encode`] and
668
+ [`PreTrainedTokenizer.__call__`] for details.
669
+
670
+ [What are input IDs?](../glossary#input-ids)
671
+ attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
672
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
673
+
674
+ - 1 for tokens that are **not masked**,
675
+ - 0 for tokens that are **masked**.
676
+
677
+ [What are attention masks?](../glossary#attention-mask)
678
+ token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
679
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
680
+
681
+ - 0 corresponds to a *sentence A* token,
682
+ - 1 corresponds to a *sentence B* token.
683
+
684
+ [What are token type IDs?](../glossary#token-type-ids)
685
+ position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
686
+ Indices of positions of each input sequence tokens in the position embeddings.
687
+ Selected in the range `[0, config.max_position_embeddings - 1]`.
688
+
689
+ [What are position IDs?](../glossary#position-ids)
690
+ head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
691
+ Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
692
+
693
+ - 1 indicates the head is **not masked**,
694
+ - 0 indicates the head is **masked**.
695
+
696
+ inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
697
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
698
+ This is useful if you want more control over how to convert *input_ids* indices into associated vectors
699
+ than the model's internal embedding lookup matrix.
700
+ output_attentions (`bool`, *optional*):
701
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
702
+ tensors for more detail.
703
+ output_hidden_states (`bool`, *optional*):
704
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
705
+ more detail.
706
+ return_dict (`bool`, *optional*):
707
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
708
+ """
709
+
710
+
711
+ @add_start_docstrings(
712
+ "The bare GLM Model transformer outputting raw hidden-states without any specific head on top.",
713
+ GLM_START_DOCSTRING,
714
+ )
715
+ class GLMModel(GLMPreTrainedModel):
716
+ """
717
+
718
+ The model can behave as an encoder (with only self-attention) as well
719
+ as a decoder, in which case a layer of cross-attention is added between
720
+ the self-attention layers, following the architecture described in [Attention is
721
+ all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani,
722
+ Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
723
+
724
+ To behave as an decoder the model needs to be initialized with the
725
+ `is_decoder` argument of the configuration set to `True`.
726
+ To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder`
727
+ argument and `add_cross_attention` set to `True`; an
728
+ `encoder_hidden_states` is then expected as an input to the forward pass.
729
+ """
730
+
731
+ def __init__(self, config):
732
+ super().__init__(config)
733
+ self.config = config
734
+ self.output_predict = config.output_predict
735
+ # Word embeddings (parallel).
736
+ self.word_embeddings = VocabEmbedding(config)
737
+
738
+ # Transformer
739
+ self.transformer = GLMStack(config.num_layers,
740
+ config.hidden_size,
741
+ config.num_attention_heads,
742
+ config.max_sequence_length,
743
+ config.embedding_dropout_prob,
744
+ config.attention_dropout_prob,
745
+ config.output_dropout_prob,
746
+ config.checkpoint_activations,
747
+ config.checkpoint_num_layers,
748
+ attention_scale=config.attention_scale,
749
+ block_position_encoding=config.block_position_encoding)
750
+
751
+ # Initialize weights and apply final processing
752
+ self.post_init()
753
+
754
+ @add_start_docstrings_to_model_forward(GLM_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
755
+ @add_code_sample_docstrings(
756
+ processor_class=_TOKENIZER_FOR_DOC,
757
+ checkpoint=_CHECKPOINT_FOR_DOC,
758
+ output_type=BaseModelOutputWithPastAndCrossAttentions,
759
+ config_class=_CONFIG_FOR_DOC,
760
+ )
761
+ def forward(
762
+ self,
763
+ input_ids=None,
764
+ position_ids=None,
765
+ attention_mask=None,
766
+ mems=None,
767
+ **kwargs
768
+ ):
769
+ batch_size = input_ids.size(0)
770
+ words_embeddings = self.word_embeddings(input_ids)
771
+ embeddings = words_embeddings
772
+
773
+ device = input_ids.device
774
+ input_shape = input_ids.size()
775
+
776
+ if position_ids is None:
777
+ position_ids = torch.arange(0, input_shape[-1], dtype=torch.long, device=device)
778
+ block_position_ids = torch.zeros(input_shape[-1], dtype=torch.long, device=device)
779
+ position_ids = torch.stack((position_ids, block_position_ids), dim=0).unsqueeze(0)
780
+ if attention_mask is None:
781
+ attention_mask = torch.zeros(batch_size)
782
+ # Transformer.
783
+ transformer_output = self.transformer(embeddings, position_ids, attention_mask, mems)
784
+ logits, hidden_layers = transformer_output
785
+ # outputs = hidden_layers
786
+ if self.output_predict:
787
+ # Parallel logits.
788
+ # logits_parallel = mpu.copy_to_model_parallel_region(
789
+ # logits)
790
+ logits = F.linear(logits, self.word_embeddings.weight)
791
+
792
+ return ModelOutput(
793
+ logits=logits,
794
+ mems=hidden_layers,
795
+ )
796
+
797
+
798
+ @add_start_docstrings(
799
+ """GLM Model transformer with a `language modeling` head on top""",
800
+ GLM_START_DOCSTRING,
801
+ )
802
+ class GLMForConditionalGeneration(GLMPreTrainedModel):
803
+ def __init__(self, config):
804
+ super().__init__(config)
805
+ self.glm = GLMModel(config)
806
+ self.post_init()
807
+
808
+ def _reorder_cache(self, past, beam_idx):
809
+ # if decoder past is not included in output
810
+ # speedy decoding is disabled and no need to reorder
811
+ if past is None:
812
+ return past
813
+ reordered_decoder_past = ()
814
+ for layer_past_states in past:
815
+ # get the correct batch idx from layer past batch dim
816
+ reordered_decoder_past = reordered_decoder_past + (
817
+ layer_past_states.index_select(0, beam_idx.to(layer_past_states.device)),)
818
+ return reordered_decoder_past
819
+
820
+ def prepare_inputs_for_generation(self, input_ids, past=None, position_ids=None, generation_attention_mask=None,
821
+ **kwargs):
822
+ # only last token for inputs_ids if past is defined in kwargs
823
+ attention_mask = generation_attention_mask
824
+ seq_length = input_ids.shape[1]
825
+ if past:
826
+ if position_ids is not None:
827
+ position_ids = position_ids[:, :, seq_length - 1].unsqueeze(-1)
828
+ if attention_mask is not None:
829
+ attention_mask = attention_mask[:, :, seq_length - 1, :seq_length].unsqueeze(-2)
830
+ input_ids = input_ids[:, -1].unsqueeze(-1)
831
+ else:
832
+ if position_ids is not None:
833
+ position_ids = position_ids[:, :, :seq_length]
834
+ if attention_mask is not None:
835
+ attention_mask = attention_mask[:, :, :seq_length, :seq_length]
836
+ return {
837
+ "input_ids": input_ids,
838
+ "position_ids": position_ids,
839
+ "attention_mask": attention_mask,
840
+ "mems": past,
841
+ }
842
+
843
+ def forward(
844
+ self,
845
+ input_ids=None,
846
+ position_ids=None,
847
+ attention_mask=None,
848
+ mems=None,
849
+ **kwargs
850
+ ):
851
+ return self.glm.forward(input_ids, position_ids, attention_mask, mems=mems, **kwargs)
852
+
853
+
854
+ @add_start_docstrings(
855
+ """GLM Model transformer with a sequence classification/regression head on top (a linear layer on top of
856
+ the pooled output) e.g. for GLUE tasks. """,
857
+ GLM_START_DOCSTRING,
858
+ )
859
+ class GLMForSequenceClassification(GLMPreTrainedModel):
860
+ def __init__(self, config, hidden_dropout=False, num_class=1):
861
+ super().__init__(config)
862
+ self.pool_token = config.pool_token
863
+ self.glm = GLMModel(config)
864
+ self.glm.output_predict = False
865
+ self.num_class = num_class
866
+ # Multi-choice head.
867
+ self.pool_layer = torch.nn.Linear(config.hidden_size, config.hidden_size)
868
+ self.multichoice_dropout = torch.nn.Dropout(hidden_dropout)
869
+ self.multichoice_head = torch.nn.Linear(config.hidden_size, num_class)
870
+
871
+ # Initialize weights and apply final processing
872
+ self.post_init()
873
+
874
+ @add_start_docstrings_to_model_forward(GLM_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
875
+ @add_code_sample_docstrings(
876
+ processor_class=_TOKENIZER_FOR_DOC,
877
+ checkpoint=_CHECKPOINT_FOR_DOC,
878
+ output_type=SequenceClassifierOutput,
879
+ config_class=_CONFIG_FOR_DOC,
880
+ )
881
+ def forward(self,
882
+ input_ids=None,
883
+ position_ids=None,
884
+ attention_mask=None,
885
+ labels=None):
886
+
887
+ num_choices = None
888
+
889
+ if len(input_ids.shape) == 3:
890
+ batch_size, num_choices = input_ids.shape[:2]
891
+ input_ids = input_ids.reshape(-1, input_ids.size(-1))
892
+ attention_mask = attention_mask.reshape(-1, *attention_mask.size()[2:])
893
+ position_ids = position_ids.reshape(-1, *position_ids.size()[2:])
894
+ model_out = self.glm.forward(input_ids, position_ids, attention_mask)
895
+ outputs, mems = model_out.last_hidden_state, model_out.hidden_states
896
+
897
+ if self.pool_token == 'start':
898
+ output = outputs[
899
+ torch.arange(outputs.size(0), dtype=attention_mask.dtype, device=attention_mask.device), attention_mask]
900
+ elif self.pool_token == 'pad':
901
+ output = outputs[torch.arange(outputs.size(0), dtype=attention_mask.dtype,
902
+ device=attention_mask.device), attention_mask - 1]
903
+ elif self.pool_token == 'cls':
904
+ output = outputs[:, 0]
905
+ else:
906
+ raise NotImplementedError
907
+
908
+ output = torch.tanh(self.pool_layer(output))
909
+ multichoice_output = self.multichoice_dropout(output)
910
+ logits = self.multichoice_head(multichoice_output)
911
+ loss_fct = CrossEntropyLoss()
912
+ if num_choices is not None:
913
+ logits = logits.view(-1, num_choices)
914
+ # assert (labels is not None, "labels must not None!")
915
+ loss = loss_fct(logits, labels)
916
+ # loss = F.cross_entropy(logits.contiguous().float(), labels.long())
917
+ return SequenceClassifierOutput(loss=loss,
918
+ logits=logits,
919
+ hidden_states=mems)
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4070e0d118c25d0cd97606a0c9b284e724710ac696ad681e6080307c927cb73f
3
+ size 19759386075
tokenization_glm.py ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from typing import Optional, Tuple, List
3
+ from shutil import copyfile
4
+
5
+ import torch
6
+ from transformers import PreTrainedTokenizer
7
+ from transformers.utils import logging
8
+ from transformers.tokenization_utils_base import BatchEncoding
9
+ import sentencepiece as spm
10
+
11
+ logger = logging.get_logger(__name__)
12
+ VOCAB_FILES_NAMES = {"vocab_file": "cog-pretrain.model"}
13
+
14
+
15
+ class GLMChineseTokenizer(PreTrainedTokenizer):
16
+ vocab_files_names = VOCAB_FILES_NAMES
17
+
18
+ def __init__(self, vocab_file, **kwargs):
19
+ super().__init__(**kwargs)
20
+
21
+ self.sp_model = spm.SentencePieceProcessor()
22
+ self.sp_model.Load(vocab_file)
23
+
24
+ @property
25
+ def vocab_size(self):
26
+ return len(self.sp_model)
27
+
28
+ def get_vocab(self):
29
+ vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
30
+ vocab.update(self.added_tokens_encoder)
31
+ return vocab
32
+
33
+ def _tokenize(self, text, **kwargs):
34
+ return self.sp_model.encode(text, out_type=str)
35
+
36
+ def _convert_token_to_id(self, token):
37
+ """Converts a token (str) in an id using the vocab."""
38
+ return self.sp_model.PieceToId(token)
39
+
40
+ def _convert_id_to_token(self, index):
41
+ """Converts an index (integer) in a token (str) using the vocab."""
42
+ return self.sp_model.IdToPiece(index)
43
+
44
+ def convert_tokens_to_string(self, tokens):
45
+ return self.sp_model.decode(tokens)
46
+
47
+ def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
48
+ if not os.path.isdir(save_directory):
49
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
50
+ return
51
+ out_vocab_file = os.path.join(
52
+ save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
53
+ )
54
+
55
+ if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
56
+ copyfile(self.vocab_file, out_vocab_file)
57
+ elif not os.path.isfile(self.vocab_file):
58
+ with open(out_vocab_file, "wb") as fi:
59
+ content_spiece_model = self.sp_model.serialized_model_proto()
60
+ fi.write(content_spiece_model)
61
+
62
+ return (out_vocab_file,)
63
+
64
+ @property
65
+ def sop_token(self) -> Optional[str]:
66
+ return "<|startofpiece|>"
67
+
68
+ @property
69
+ def sop_token_id(self) -> Optional[int]:
70
+ """
71
+ `Optional[int]`: Id of the start token in the vocabulary, used when training a model with autoregressive blank filling. Returns `None` if the token has not been set.
72
+ """
73
+ return self.convert_tokens_to_ids(self.sop_token)
74
+
75
+ @property
76
+ def eop_token(self) -> Optional[str]:
77
+ return "<|endofpiece|>"
78
+
79
+ @property
80
+ def eop_token_id(self) -> Optional[int]:
81
+ """
82
+ `Optional[int]`: Id of the end token in the vocabulary, used when training a model with autoregressive blank filling. Returns `None` if the token has not been set.
83
+ """
84
+ return self.convert_tokens_to_ids(self.eop_token)
85
+
86
+ @property
87
+ def gmask_token_id(self) -> int:
88
+ return self.convert_tokens_to_ids("[gMASK]")
89
+
90
+ @property
91
+ def smask_token_id(self) -> int:
92
+ return self.convert_tokens_to_ids("[sMASK]")
93
+
94
+ def build_inputs_for_generation(self, model_input: BatchEncoding, max_gen_length=512):
95
+ mask_ids = [self.mask_token_id, self.smask_token_id, self.gmask_token_id]
96
+ input_ids = model_input.input_ids
97
+ batch_size, seq_length = input_ids.shape[:2]
98
+ position_id, block_position_id = list(range(seq_length)), [0 for _ in range(seq_length)]
99
+ position_ids, block_position_ids = [], []
100
+ for i in range(batch_size):
101
+ mask_positions = []
102
+ for mask_id in mask_ids:
103
+ mask_positions += (input_ids[i] == mask_id).nonzero(as_tuple=True)[0].tolist()
104
+ if not mask_positions:
105
+ raise ValueError("Cannot find mask token in the input")
106
+ mask_positions.sort()
107
+ mask_pos = mask_positions[0]
108
+ position_ids.append(position_id + [mask_pos] * max_gen_length)
109
+ block_position_ids.append(block_position_id + list(range(1, max_gen_length + 1)))
110
+ position_ids = torch.tensor(position_ids, dtype=input_ids.dtype, device=input_ids.device)
111
+ block_position_ids = torch.tensor(block_position_ids, dtype=input_ids.dtype, device=input_ids.device)
112
+ position_ids = torch.stack((position_ids, block_position_ids), dim=1)
113
+ attention_mask = model_input.attention_mask
114
+ attention_mask = attention_mask.unsqueeze(1).expand(-1, seq_length + max_gen_length, -1)
115
+ generation_attention_mask = torch.cat([attention_mask.new_zeros((seq_length, max_gen_length)),
116
+ torch.tril(attention_mask.new_ones((max_gen_length, max_gen_length)))],
117
+ dim=0).unsqueeze(0).expand(batch_size, -1, -1)
118
+ attention_mask = torch.cat((attention_mask, generation_attention_mask), dim=2)
119
+ attention_mask = attention_mask.unsqueeze(1)
120
+ input_ids = torch.cat((input_ids, input_ids.new_full((batch_size, 1), self.sop_token_id)), dim=-1)
121
+ return BatchEncoding(
122
+ {"input_ids": input_ids, "position_ids": position_ids, "generation_attention_mask": attention_mask}
123
+ )
124
+
125
+ def build_inputs_with_special_tokens(
126
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
127
+ ) -> List[int]:
128
+ """
129
+ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
130
+ adding special tokens. A BERT sequence has the following format:
131
+
132
+ - single sequence: ``[CLS] X [SEP]``
133
+ - pair of sequences: ``[CLS] A [SEP] B [SEP]``
134
+
135
+ Args:
136
+ token_ids_0 (:obj:`List[int]`):
137
+ List of IDs to which the special tokens will be added.
138
+ token_ids_1 (:obj:`List[int]`, `optional`):
139
+ Optional second list of IDs for sequence pairs.
140
+
141
+ Returns:
142
+ :obj:`List[int]`: List of `input IDs <../glossary.html#input-ids>`__ with the appropriate special tokens.
143
+ """
144
+ assert token_ids_1 is None
145
+ cls = [self.cls_token_id]
146
+ eos = [self.eos_token_id]
147
+ return cls + token_ids_0 + eos
tokenizer_config.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name_or_path": "THUDM/glm-10b-chinese",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|endoftext|>",
5
+ "cls_token": "[CLS]",
6
+ "mask_token": "[MASK]",
7
+ "unk_token": "[UNK]",
8
+ "additional_special_tokens": ["<|startofpiece|>", "<|endofpiece|>", "[gMASK]", "[sMASK]"],
9
+ "add_prefix_space": false,
10
+ "tokenizer_class": "GLMChineseTokenizer",
11
+ "use_fast": false,
12
+ "auto_map": {
13
+ "AutoTokenizer": [
14
+ "tokenization_glm.GLMChineseTokenizer",
15
+ null
16
+ ]
17
+ }
18
+ }