Upload 7 files

models/DPCNN.py

@@ -0,0 +1,89 @@
+# coding: UTF-8
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+
+
+class Config(object):
+
+    """配置参数"""
+    def __init__(self, dataset, embedding):
+        self.model_name = 'DPCNN'
+        self.train_path = dataset + '/data/train.txt'                                # 训练集
+        self.dev_path = dataset + '/data/dev.txt'                                    # 验证集
+        self.test_path = dataset + '/data/test.txt'                                  # 测试集
+        self.class_list = [x.strip() for x in open(
+            dataset + '/data/class.txt', encoding='utf-8').readlines()]              # 类别名单
+        self.vocab_path = dataset + '/data/vocab.pkl'                                # 词表
+        self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'        # 模型训练结果
+        self.log_path = dataset + '/log/' + self.model_name
+        self.embedding_pretrained = torch.tensor(
+            np.load(dataset + '/data/' + embedding)["embeddings"].astype('float32'))\
+            if embedding != 'random' else None                                       # 预训练词向量
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')   # 设备
+
+        self.dropout = 0.5                                              # 随机失活
+        self.require_improvement = 1000                                 # 若超过1000batch效果还没提升，则提前结束训练
+        self.num_classes = len(self.class_list)                         # 类别数
+        self.n_vocab = 0                                                # 词表大小，在运行时赋值
+        self.num_epochs = 20                                            # epoch数
+        self.batch_size = 128                                           # mini-batch大小
+        self.pad_size = 32                                              # 每句话处理成的长度(短填长切)
+        self.learning_rate = 1e-3                                       # 学习率
+        self.embed = self.embedding_pretrained.size(1)\
+            if self.embedding_pretrained is not None else 300           # 字向量维度
+        self.num_filters = 250                                          # 卷积核数量(channels数)
+
+
+'''Deep Pyramid Convolutional Neural Networks for Text Categorization'''
+
+
+class Model(nn.Module):
+    def __init__(self, config):
+        super(Model, self).__init__()
+        if config.embedding_pretrained is not None:
+            self.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)
+        else:
+            self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)
+        self.conv_region = nn.Conv2d(1, config.num_filters, (3, config.embed), stride=1)
+        self.conv = nn.Conv2d(config.num_filters, config.num_filters, (3, 1), stride=1)
+        self.max_pool = nn.MaxPool2d(kernel_size=(3, 1), stride=2)
+        self.padding1 = nn.ZeroPad2d((0, 0, 1, 1))  # top bottom
+        self.padding2 = nn.ZeroPad2d((0, 0, 0, 1))  # bottom
+        self.relu = nn.ReLU()
+        self.fc = nn.Linear(config.num_filters, config.num_classes)
+
+    def forward(self, x):
+        x = x[0]
+        x = self.embedding(x)
+        x = x.unsqueeze(1)  # [batch_size, 250, seq_len, 1]
+        x = self.conv_region(x)  # [batch_size, 250, seq_len-3+1, 1]
+
+        x = self.padding1(x)  # [batch_size, 250, seq_len, 1]
+        x = self.relu(x)
+        x = self.conv(x)  # [batch_size, 250, seq_len-3+1, 1]
+        x = self.padding1(x)  # [batch_size, 250, seq_len, 1]
+        x = self.relu(x)
+        x = self.conv(x)  # [batch_size, 250, seq_len-3+1, 1]
+        while x.size()[2] > 2:
+            x = self._block(x)
+        x = x.squeeze()  # [batch_size, num_filters(250)]
+        x = self.fc(x)
+        return x
+
+    def _block(self, x):
+        x = self.padding2(x)
+        px = self.max_pool(x)
+
+        x = self.padding1(px)
+        x = F.relu(x)
+        x = self.conv(x)
+
+        x = self.padding1(x)
+        x = F.relu(x)
+        x = self.conv(x)
+
+        # Short Cut
+        x = x + px
+        return x

models/FastText.py

@@ -0,0 +1,69 @@
+# coding: UTF-8
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+
+
+class Config(object):
+
+    """配置参数"""
+    def __init__(self, dataset, embedding):
+        self.model_name = 'FastText'
+        self.train_path = dataset + '/data/train.txt'                                # 训练集
+        self.dev_path = dataset + '/data/dev.txt'                                    # 验证集
+        self.test_path = dataset + '/data/test.txt'                                  # 测试集
+        self.class_list = [x.strip() for x in open(
+            dataset + '/data/class.txt', encoding='utf-8').readlines()]              # 类别名单
+        self.vocab_path = dataset + '/data/vocab.pkl'                                # 词表
+        self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'        # 模型训练结果
+        self.log_path = dataset + '/log/' + self.model_name
+        self.embedding_pretrained = torch.tensor(
+            np.load(dataset + '/data/' + embedding)["embeddings"].astype('float32'))\
+            if embedding != 'random' else None                                       # 预训练词向量
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')   # 设备
+
+        self.dropout = 0.5                                              # 随机失活
+        self.require_improvement = 1000                                 # 若超过1000batch效果还没提升，则提前结束训练
+        self.num_classes = len(self.class_list)                         # 类别数
+        self.n_vocab = 0                                                # 词表大小，在运行时赋值
+        self.num_epochs = 20                                            # epoch数
+        self.batch_size = 128                                           # mini-batch大小
+        self.pad_size = 32                                              # 每句话处理成的长度(短填长切)
+        self.learning_rate = 1e-3                                       # 学习率
+        self.embed = self.embedding_pretrained.size(1)\
+            if self.embedding_pretrained is not None else 300           # 字向量维度
+        self.hidden_size = 256                                          # 隐藏层大小
+        self.n_gram_vocab = 250499                                      # ngram 词表大小
+
+
+'''Bag of Tricks for Efficient Text Classification'''
+
+
+class Model(nn.Module):
+    def __init__(self, config):
+        super(Model, self).__init__()
+        if config.embedding_pretrained is not None:
+            self.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)
+        else:
+            self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)
+        self.embedding_ngram2 = nn.Embedding(config.n_gram_vocab, config.embed)
+        self.embedding_ngram3 = nn.Embedding(config.n_gram_vocab, config.embed)
+        self.dropout = nn.Dropout(config.dropout)
+        self.fc1 = nn.Linear(config.embed * 3, config.hidden_size)
+        # self.dropout2 = nn.Dropout(config.dropout)
+        self.fc2 = nn.Linear(config.hidden_size, config.num_classes)
+
+    def forward(self, x):
+
+        out_word = self.embedding(x[0])
+        out_bigram = self.embedding_ngram2(x[2])
+        out_trigram = self.embedding_ngram3(x[3])
+        out = torch.cat((out_word, out_bigram, out_trigram), -1)
+
+        out = out.mean(dim=1)
+        out = self.dropout(out)
+        out = self.fc1(out)
+        out = F.relu(out)
+        out = self.fc2(out)
+        return out

models/TextCNN.py

@@ -0,0 +1,67 @@
+# coding: UTF-8
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+
+
+class Config(object):
+
+    """配置参数"""
+    def __init__(self, dataset, embedding):
+        self.model_name = 'TextCNN'
+        self.train_path = dataset + '/data/train.txt'                                # 训练集
+        self.dev_path = dataset + '/data/dev.txt'                                    # 验证集
+        self.test_path = dataset + '/data/test.txt'                                  # 测试集
+        self.class_list = [x.strip() for x in open(
+            dataset + '/data/class.txt', encoding='utf-8').readlines()]              # 类别名单
+        self.vocab_path = dataset + '/data/vocab.pkl'                                # 词表
+        self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'        # 模型训练结果
+        self.log_path = dataset + '/log/' + self.model_name
+        self.embedding_pretrained = torch.tensor(
+            np.load(dataset + '/data/' + embedding)["embeddings"].astype('float32'))\
+            if embedding != 'random' else None                                       # 预训练词向量
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')   # 设备
+
+        self.dropout = 0.5                                              # 随机失活
+        self.require_improvement = 1000                                 # 若超过1000batch效果还没提升，则提前结束训练
+        self.num_classes = len(self.class_list)                         # 类别数
+        self.n_vocab = 0                                                # 词表大小，在运行时赋值
+        self.num_epochs = 20                                            # epoch数
+        self.batch_size = 1                                           # mini-batch大小
+        self.pad_size = 32                                              # 每句话处理成的长度(短填长切)
+        self.learning_rate = 1e-3                                       # 学习率
+        self.embed = self.embedding_pretrained.size(1)\
+            if self.embedding_pretrained is not None else 300           # 字向量维度
+        self.filter_sizes = (2, 3, 4)                                   # 卷积核尺寸
+        self.num_filters = 256                                          # 卷积核数量(channels数)
+
+
+'''Convolutional Neural Networks for Sentence Classification'''
+
+
+class Model(nn.Module):
+    def __init__(self, config):
+        super(Model, self).__init__()
+        if config.embedding_pretrained is not None:
+            self.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)
+        else:
+            self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)
+        self.convs = nn.ModuleList(
+            [nn.Conv2d(1, config.num_filters, (k, config.embed)) for k in config.filter_sizes])
+        self.dropout = nn.Dropout(config.dropout)
+        self.fc = nn.Linear(config.num_filters * len(config.filter_sizes), config.num_classes)
+
+    def conv_and_pool(self, x, conv):
+        x = F.relu(conv(x)).squeeze(3)
+        x = F.max_pool1d(x, x.size(2)).squeeze(2)
+        return x
+
+    def forward(self, x):
+        out = self.embedding(x[0])
+        out = out.unsqueeze(1)
+        out = torch.cat([self.conv_and_pool(out, conv) for conv in self.convs], 1)
+        out = self.dropout(out)
+        out = self.fc(out)
+        return out
+

models/TextRCNN.py

@@ -0,0 +1,64 @@
+# coding: UTF-8
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+
+
+class Config(object):
+
+    """配置参数"""
+    def __init__(self, dataset, embedding):
+        self.model_name = 'TextRCNN'
+        self.train_path = dataset + '/data/train.txt'                                # 训练集
+        self.dev_path = dataset + '/data/dev.txt'                                    # 验证集
+        self.test_path = dataset + '/data/test.txt'                                  # 测试集
+        self.class_list = [x.strip() for x in open(
+            dataset + '/data/class.txt', encoding='utf-8').readlines()]              # 类别名单
+        self.vocab_path = dataset + '/data/vocab.pkl'                                # 词表
+        self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'        # 模型训练结果
+        self.log_path = dataset + '/log/' + self.model_name
+        self.embedding_pretrained = torch.tensor(
+            np.load(dataset + '/data/' + embedding)["embeddings"].astype('float32'))\
+            if embedding != 'random' else None                                       # 预训练词向量
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')   # 设备
+
+        self.dropout = 1.0                                              # 随机失活
+        self.require_improvement = 1000                                 # 若超过1000batch效果还没提升，则提前结束训练
+        self.num_classes = len(self.class_list)                         # 类别数
+        self.n_vocab = 0                                                # 词表大小，在运行时赋值
+        self.num_epochs = 10                                            # epoch数
+        self.batch_size = 128                                           # mini-batch大小
+        self.pad_size = 32                                              # 每句话处理成的长度(短填长切)
+        self.learning_rate = 1e-3                                       # 学习率
+        self.embed = self.embedding_pretrained.size(1)\
+            if self.embedding_pretrained is not None else 300           # 字向量维度, 若使用了预训练词向量，则维度统一
+        self.hidden_size = 256                                          # lstm隐藏层
+        self.num_layers = 1                                             # lstm层数
+
+
+'''Recurrent Convolutional Neural Networks for Text Classification'''
+
+
+class Model(nn.Module):
+    def __init__(self, config):
+        super(Model, self).__init__()
+        if config.embedding_pretrained is not None:
+            self.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)
+        else:
+            self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)
+        self.lstm = nn.LSTM(config.embed, config.hidden_size, config.num_layers,
+                            bidirectional=True, batch_first=True, dropout=config.dropout)
+        self.maxpool = nn.MaxPool1d(config.pad_size)
+        self.fc = nn.Linear(config.hidden_size * 2 + config.embed, config.num_classes)
+
+    def forward(self, x):
+        x, _ = x
+        embed = self.embedding(x)  # [batch_size, seq_len, embeding]=[64, 32, 64]
+        out, _ = self.lstm(embed)
+        out = torch.cat((embed, out), 2)
+        out = F.relu(out)
+        out = out.permute(0, 2, 1)
+        out = self.maxpool(out).squeeze()
+        out = self.fc(out)
+        return out

models/TextRNN.py

@@ -0,0 +1,75 @@
+# coding: UTF-8
+import torch
+import torch.nn as nn
+import numpy as np
+
+
+class Config(object):
+
+    """配置参数"""
+    def __init__(self, dataset, embedding):
+        self.model_name = 'TextRNN'
+        self.train_path = dataset + '/data/train.txt'                                # 训练集
+        self.dev_path = dataset + '/data/dev.txt'                                    # 验证集
+        self.test_path = dataset + '/data/test.txt'                                  # 测试集
+        self.class_list = [x.strip() for x in open(
+            dataset + '/data/class.txt', encoding='utf-8').readlines()]              # 类别名单
+        self.vocab_path = dataset + '/data/vocab.pkl'                                # 词表
+        self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'        # 模型训练结果
+        self.log_path = dataset + '/log/' + self.model_name
+        self.embedding_pretrained = torch.tensor(
+            np.load(dataset + '/data/' + embedding)["embeddings"].astype('float32'))\
+            if embedding != 'random' else None                                       # 预训练词向量
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')   # 设备
+
+        self.dropout = 0.5                                              # 随机失活
+        self.require_improvement = 1000                                 # 若超过1000batch效果还没提升，则提前结束训练
+        self.num_classes = len(self.class_list)                         # 类别数
+        self.n_vocab = 0                                                # 词表大小，在运行时赋值
+        self.num_epochs = 10                                            # epoch数
+        self.batch_size = 128                                           # mini-batch大小
+        self.pad_size = 32                                              # 每句话处理成的长度(短填长切)
+        self.learning_rate = 1e-3                                       # 学习率
+        self.embed = self.embedding_pretrained.size(1)\
+            if self.embedding_pretrained is not None else 300           # 字向量维度, 若使用了预训练词向量，则维度统一
+        self.hidden_size = 128                                          # lstm隐藏层
+        self.num_layers = 2                                             # lstm层数
+
+
+'''Recurrent Neural Network for Text Classification with Multi-Task Learning'''
+
+
+class Model(nn.Module):
+    def __init__(self, config):
+        super(Model, self).__init__()
+        if config.embedding_pretrained is not None:
+            self.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)
+        else:
+            self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)
+        self.lstm = nn.LSTM(config.embed, config.hidden_size, config.num_layers,
+                            bidirectional=True, batch_first=True, dropout=config.dropout)
+        self.fc = nn.Linear(config.hidden_size * 2, config.num_classes)
+
+    def forward(self, x):
+        x, _ = x
+        out = self.embedding(x)  # [batch_size, seq_len, embeding]=[128, 32, 300]
+        out, _ = self.lstm(out)
+        out = self.fc(out[:, -1, :])  # 句子最后时刻的 hidden state
+        return out
+
+    '''变长RNN，效果差不多，甚至还低了点...'''
+    # def forward(self, x):
+    #     x, seq_len = x
+    #     out = self.embedding(x)
+    #     _, idx_sort = torch.sort(seq_len, dim=0, descending=True)  # 长度从长到短排序（index）
+    #     _, idx_unsort = torch.sort(idx_sort)  # 排序后，原序列的 index
+    #     out = torch.index_select(out, 0, idx_sort)
+    #     seq_len = list(seq_len[idx_sort])
+    #     out = nn.utils.rnn.pack_padded_sequence(out, seq_len, batch_first=True)
+    #     # [batche_size, seq_len, num_directions * hidden_size]
+    #     out, (hn, _) = self.lstm(out)
+    #     out = torch.cat((hn[2], hn[3]), -1)
+    #     # out, _ = nn.utils.rnn.pad_packed_sequence(out, batch_first=True)
+    #     out = out.index_select(0, idx_unsort)
+    #     out = self.fc(out)
+    #     return out

models/TextRNN_Att.py

@@ -0,0 +1,73 @@
+# coding: UTF-8
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+
+
+class Config(object):
+
+    """配置参数"""
+    def __init__(self, dataset, embedding):
+        self.model_name = 'TextRNN_Att'
+        self.train_path = dataset + '/data/train.txt'                                # 训练集
+        self.dev_path = dataset + '/data/dev.txt'                                    # 验证集
+        self.test_path = dataset + '/data/test.txt'                                  # 测试集
+        self.class_list = [x.strip() for x in open(
+            dataset + '/data/class.txt', encoding='utf-8').readlines()]              # 类别名单
+        self.vocab_path = dataset + '/data/vocab.pkl'                                # 词表
+        self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'        # 模型训练结果
+        self.log_path = dataset + '/log/' + self.model_name
+        self.embedding_pretrained = torch.tensor(
+            np.load(dataset + '/data/' + embedding)["embeddings"].astype('float32'))\
+            if embedding != 'random' else None                                       # 预训练词向量
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')   # 设备
+
+        self.dropout = 0.5                                              # 随机失活
+        self.require_improvement = 1000                                 # 若超过1000batch效果还没提升，则提前结束训练
+        self.num_classes = len(self.class_list)                         # 类别数
+        self.n_vocab = 0                                                # 词表大小，在运行时赋值
+        self.num_epochs = 10                                            # epoch数
+        self.batch_size = 128                                           # mini-batch大小
+        self.pad_size = 32                                              # 每句话处理成的长度(短填长切)
+        self.learning_rate = 1e-3                                       # 学习率
+        self.embed = self.embedding_pretrained.size(1)\
+            if self.embedding_pretrained is not None else 300           # 字向量维度, 若使用了预训练词向量，则维度统一
+        self.hidden_size = 128                                          # lstm隐藏层
+        self.num_layers = 2                                             # lstm层数
+        self.hidden_size2 = 64
+
+
+'''Attention-Based Bidirectional Long Short-Term Memory Networks for Relation Classification'''
+
+
+class Model(nn.Module):
+    def __init__(self, config):
+        super(Model, self).__init__()
+        if config.embedding_pretrained is not None:
+            self.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)
+        else:
+            self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)
+        self.lstm = nn.LSTM(config.embed, config.hidden_size, config.num_layers,
+                            bidirectional=True, batch_first=True, dropout=config.dropout)
+        self.tanh1 = nn.Tanh()
+        # self.u = nn.Parameter(torch.Tensor(config.hidden_size * 2, config.hidden_size * 2))
+        self.w = nn.Parameter(torch.zeros(config.hidden_size * 2))
+        self.tanh2 = nn.Tanh()
+        self.fc1 = nn.Linear(config.hidden_size * 2, config.hidden_size2)
+        self.fc = nn.Linear(config.hidden_size2, config.num_classes)
+
+    def forward(self, x):
+        x, _ = x
+        emb = self.embedding(x)  # [batch_size, seq_len, embeding]=[128, 32, 300]
+        H, _ = self.lstm(emb)  # [batch_size, seq_len, hidden_size * num_direction]=[128, 32, 256]
+
+        M = self.tanh1(H)  # [128, 32, 256]
+        # M = torch.tanh(torch.matmul(H, self.u))
+        alpha = F.softmax(torch.matmul(M, self.w), dim=1).unsqueeze(-1)  # [128, 32, 1]
+        out = H * alpha  # [128, 32, 256]
+        out = torch.sum(out, 1)  # [128, 256]
+        out = F.relu(out)
+        out = self.fc1(out)
+        out = self.fc(out)  # [128, 64]
+        return out

models/Transformer.py

@@ -0,0 +1,178 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+import copy
+
+
+class Config(object):
+
+    """配置参数"""
+    def __init__(self, dataset, embedding):
+        self.model_name = 'Transformer'
+        self.train_path = dataset + '/data/train.txt'                                # 训练集
+        self.dev_path = dataset + '/data/dev.txt'                                    # 验证集
+        self.test_path = dataset + '/data/test.txt'                                  # 测试集
+        self.class_list = [x.strip() for x in open(
+            dataset + '/data/class.txt', encoding='utf-8').readlines()]              # 类别名单
+        self.vocab_path = dataset + '/data/vocab.pkl'                                # 词表
+        self.save_path = dataset + '/saved_dict/' + self.model_name + '.ckpt'        # 模型训练结果
+        self.log_path = dataset + '/log/' + self.model_name
+        self.embedding_pretrained = torch.tensor(
+            np.load(dataset + '/data/' + embedding)["embeddings"].astype('float32'))\
+            if embedding != 'random' else None                                       # 预训练词向量
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')   # 设备
+
+        self.dropout = 0.5                                              # 随机失活
+        self.require_improvement = 2000                                 # 若超过1000batch效果还没提升，则提前结束训练
+        self.num_classes = len(self.class_list)                         # 类别数
+        self.n_vocab = 0                                                # 词表大小，在运行时赋值
+        self.num_epochs = 20                                            # epoch数
+        self.batch_size = 128                                           # mini-batch大小
+        self.pad_size = 32                                              # 每句话处理成的长度(短填长切)
+        self.learning_rate = 5e-4                                       # 学习率
+        self.embed = self.embedding_pretrained.size(1)\
+            if self.embedding_pretrained is not None else 300           # 字向量维度
+        self.dim_model = 300
+        self.hidden = 1024
+        self.last_hidden = 512
+        self.num_head = 5
+        self.num_encoder = 2
+
+
+'''Attention Is All You Need'''
+
+
+class Model(nn.Module):
+    def __init__(self, config):
+        super(Model, self).__init__()
+        if config.embedding_pretrained is not None:
+            self.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)
+        else:
+            self.embedding = nn.Embedding(config.n_vocab, config.embed, padding_idx=config.n_vocab - 1)
+
+        self.postion_embedding = Positional_Encoding(config.embed, config.pad_size, config.dropout, config.device)
+        self.encoder = Encoder(config.dim_model, config.num_head, config.hidden, config.dropout)
+        self.encoders = nn.ModuleList([
+            copy.deepcopy(self.encoder)
+            # Encoder(config.dim_model, config.num_head, config.hidden, config.dropout)
+            for _ in range(config.num_encoder)])
+
+        self.fc1 = nn.Linear(config.pad_size * config.dim_model, config.num_classes)
+        # self.fc2 = nn.Linear(config.last_hidden, config.num_classes)
+        # self.fc1 = nn.Linear(config.dim_model, config.num_classes)
+
+    def forward(self, x):
+        out = self.embedding(x[0])
+        out = self.postion_embedding(out)
+        for encoder in self.encoders:
+            out = encoder(out)
+        out = out.view(out.size(0), -1)
+        # out = torch.mean(out, 1)
+        out = self.fc1(out)
+        return out
+
+
+class Encoder(nn.Module):
+    def __init__(self, dim_model, num_head, hidden, dropout):
+        super(Encoder, self).__init__()
+        self.attention = Multi_Head_Attention(dim_model, num_head, dropout)
+        self.feed_forward = Position_wise_Feed_Forward(dim_model, hidden, dropout)
+
+    def forward(self, x):
+        out = self.attention(x)
+        out = self.feed_forward(out)
+        return out
+
+
+class Positional_Encoding(nn.Module):
+    def __init__(self, embed, pad_size, dropout, device):
+        super(Positional_Encoding, self).__init__()
+        self.device = device
+        self.pe = torch.tensor([[pos / (10000.0 ** (i // 2 * 2.0 / embed)) for i in range(embed)] for pos in range(pad_size)])
+        self.pe[:, 0::2] = np.sin(self.pe[:, 0::2])
+        self.pe[:, 1::2] = np.cos(self.pe[:, 1::2])
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x):
+        out = x + nn.Parameter(self.pe, requires_grad=False).to(self.device)
+        out = self.dropout(out)
+        return out
+
+
+class Scaled_Dot_Product_Attention(nn.Module):
+    '''Scaled Dot-Product Attention '''
+    def __init__(self):
+        super(Scaled_Dot_Product_Attention, self).__init__()
+
+    def forward(self, Q, K, V, scale=None):
+        '''
+        Args:
+            Q: [batch_size, len_Q, dim_Q]
+            K: [batch_size, len_K, dim_K]
+            V: [batch_size, len_V, dim_V]
+            scale: 缩放因子 论文为根号dim_K
+        Return:
+            self-attention后的张量，以及attention张量
+        '''
+        attention = torch.matmul(Q, K.permute(0, 2, 1))
+        if scale:
+            attention = attention * scale
+        # if mask:  # TODO change this
+        #     attention = attention.masked_fill_(mask == 0, -1e9)
+        attention = F.softmax(attention, dim=-1)
+        context = torch.matmul(attention, V)
+        return context
+
+
+class Multi_Head_Attention(nn.Module):
+    def __init__(self, dim_model, num_head, dropout=0.0):
+        super(Multi_Head_Attention, self).__init__()
+        self.num_head = num_head
+        assert dim_model % num_head == 0
+        self.dim_head = dim_model // self.num_head
+        self.fc_Q = nn.Linear(dim_model, num_head * self.dim_head)
+        self.fc_K = nn.Linear(dim_model, num_head * self.dim_head)
+        self.fc_V = nn.Linear(dim_model, num_head * self.dim_head)
+        self.attention = Scaled_Dot_Product_Attention()
+        self.fc = nn.Linear(num_head * self.dim_head, dim_model)
+        self.dropout = nn.Dropout(dropout)
+        self.layer_norm = nn.LayerNorm(dim_model)
+
+    def forward(self, x):
+        batch_size = x.size(0)
+        Q = self.fc_Q(x)
+        K = self.fc_K(x)
+        V = self.fc_V(x)
+        Q = Q.view(batch_size * self.num_head, -1, self.dim_head)
+        K = K.view(batch_size * self.num_head, -1, self.dim_head)
+        V = V.view(batch_size * self.num_head, -1, self.dim_head)
+        # if mask:  # TODO
+        #     mask = mask.repeat(self.num_head, 1, 1)  # TODO change this
+        scale = K.size(-1) ** -0.5  # 缩放因子
+        context = self.attention(Q, K, V, scale)
+
+        context = context.view(batch_size, -1, self.dim_head * self.num_head)
+        out = self.fc(context)
+        out = self.dropout(out)
+        out = out + x  # 残差连接
+        out = self.layer_norm(out)
+        return out
+
+
+class Position_wise_Feed_Forward(nn.Module):
+    def __init__(self, dim_model, hidden, dropout=0.0):
+        super(Position_wise_Feed_Forward, self).__init__()
+        self.fc1 = nn.Linear(dim_model, hidden)
+        self.fc2 = nn.Linear(hidden, dim_model)
+        self.dropout = nn.Dropout(dropout)
+        self.layer_norm = nn.LayerNorm(dim_model)
+
+    def forward(self, x):
+        out = self.fc1(x)
+        out = F.relu(out)
+        out = self.fc2(out)
+        out = self.dropout(out)
+        out = out + x  # 残差连接
+        out = self.layer_norm(out)
+        return out