SkywalkerLu
commited on
Commit
•
29c8230
1
Parent(s):
90a0a21
Upload model
Browse files- config.json +24 -0
- configuration_TransHLA_II.py +78 -0
- modeling_TransHLA_II.py +149 -0
- pytorch_model.bin +3 -0
config.json
ADDED
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"architectures": [
|
3 |
+
"TransHLA_II_Model"
|
4 |
+
],
|
5 |
+
"auto_map": {
|
6 |
+
"AutoConfig": "configuration_TransHLA_II.TransHLA_II_Config",
|
7 |
+
"AutoModel": "modeling_TransHLA_II.TransHLA_II_Model"
|
8 |
+
},
|
9 |
+
"cnn_kernel_size": 3,
|
10 |
+
"cnn_num_channel": 256,
|
11 |
+
"cnn_padding_index": 0,
|
12 |
+
"cnn_padding_size": 1,
|
13 |
+
"cnn_stride": 1,
|
14 |
+
"d_ff": 64,
|
15 |
+
"d_model": 1280,
|
16 |
+
"max_len": 21,
|
17 |
+
"model_type": "TransHLA",
|
18 |
+
"n_head": 8,
|
19 |
+
"n_layers": 6,
|
20 |
+
"pooling_size": 2,
|
21 |
+
"region_embedding_size": 3,
|
22 |
+
"torch_dtype": "float32",
|
23 |
+
"transformers_version": "4.24.0"
|
24 |
+
}
|
configuration_TransHLA_II.py
ADDED
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import torch
|
2 |
+
import torch.nn as nn
|
3 |
+
import torch.nn.functional as F
|
4 |
+
import esm
|
5 |
+
import numpy as np
|
6 |
+
import pandas as pd
|
7 |
+
from sklearn.model_selection import KFold, StratifiedShuffleSplit, StratifiedKFold
|
8 |
+
import collections
|
9 |
+
from torch.utils.data import DataLoader, TensorDataset
|
10 |
+
import os
|
11 |
+
from sklearn.metrics import roc_curve, roc_auc_score
|
12 |
+
from sklearn.metrics import precision_recall_curve, average_precision_score
|
13 |
+
from sklearn.metrics import matthews_corrcoef
|
14 |
+
from sklearn.metrics import f1_score
|
15 |
+
from sklearn.metrics import recall_score, precision_score
|
16 |
+
import random
|
17 |
+
from sklearn.metrics import auc
|
18 |
+
from sklearn.decomposition import PCA
|
19 |
+
import matplotlib.pyplot as plt
|
20 |
+
#import esm
|
21 |
+
from tqdm import tqdm
|
22 |
+
import time
|
23 |
+
import seaborn as sns
|
24 |
+
from sklearn.metrics import confusion_matrix, precision_recall_curve, average_precision_score, matthews_corrcoef, recall_score, f1_score, precision_score
|
25 |
+
torch.backends.cudnn.enabled = True
|
26 |
+
torch.backends.cudnn.benchmark = True
|
27 |
+
|
28 |
+
|
29 |
+
|
30 |
+
|
31 |
+
|
32 |
+
|
33 |
+
from transformers import PretrainedConfig
|
34 |
+
from typing import List
|
35 |
+
|
36 |
+
|
37 |
+
class TransHLA_II_Config(PretrainedConfig):
|
38 |
+
model_type = "TransHLA"
|
39 |
+
|
40 |
+
def __init__(
|
41 |
+
self,
|
42 |
+
max_len = 21,
|
43 |
+
n_layers = 6,
|
44 |
+
n_head = 8,
|
45 |
+
d_model = 1280,
|
46 |
+
d_ff = 64,
|
47 |
+
cnn_padding_index = 0,
|
48 |
+
cnn_num_channel = 256,
|
49 |
+
region_embedding_size = 3,
|
50 |
+
cnn_kernel_size = 3,
|
51 |
+
cnn_padding_size = 1,
|
52 |
+
cnn_stride = 1,
|
53 |
+
pooling_size = 2,
|
54 |
+
**kwargs,
|
55 |
+
):
|
56 |
+
|
57 |
+
|
58 |
+
self.max_len = max_len
|
59 |
+
self.n_layers = n_layers
|
60 |
+
self.n_head = n_head
|
61 |
+
self.d_model = d_model
|
62 |
+
self.d_ff = d_ff
|
63 |
+
self.cnn_padding_index = cnn_padding_index
|
64 |
+
self.cnn_num_channel = cnn_num_channel
|
65 |
+
self.region_embedding_size = region_embedding_size
|
66 |
+
self.cnn_kernel_size= cnn_kernel_size
|
67 |
+
self.cnn_padding_size = cnn_padding_size
|
68 |
+
self.cnn_stride = cnn_stride
|
69 |
+
self.pooling_size = pooling_size
|
70 |
+
super().__init__(**kwargs)
|
71 |
+
|
72 |
+
|
73 |
+
|
74 |
+
resnet50d_config = TransHLA_II_Config()
|
75 |
+
resnet50d_config.save_pretrained("TransHLA_II")
|
76 |
+
|
77 |
+
|
78 |
+
|
modeling_TransHLA_II.py
ADDED
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import torch
|
2 |
+
import torch.nn as nn
|
3 |
+
import torch.nn.functional as F
|
4 |
+
import esm
|
5 |
+
import numpy as np
|
6 |
+
import pandas as pd
|
7 |
+
from sklearn.model_selection import KFold, StratifiedShuffleSplit, StratifiedKFold
|
8 |
+
import collections
|
9 |
+
from torch.utils.data import DataLoader, TensorDataset
|
10 |
+
import os
|
11 |
+
from sklearn.metrics import roc_curve, roc_auc_score
|
12 |
+
from sklearn.metrics import precision_recall_curve, average_precision_score
|
13 |
+
from sklearn.metrics import matthews_corrcoef
|
14 |
+
from sklearn.metrics import f1_score
|
15 |
+
from sklearn.metrics import recall_score, precision_score
|
16 |
+
import random
|
17 |
+
from sklearn.metrics import auc
|
18 |
+
from sklearn.decomposition import PCA
|
19 |
+
import matplotlib.pyplot as plt
|
20 |
+
#import esm
|
21 |
+
from tqdm import tqdm
|
22 |
+
import time
|
23 |
+
import seaborn as sns
|
24 |
+
from sklearn.metrics import confusion_matrix, precision_recall_curve, average_precision_score, matthews_corrcoef, recall_score, f1_score, precision_score
|
25 |
+
torch.backends.cudnn.enabled = True
|
26 |
+
torch.backends.cudnn.benchmark = True
|
27 |
+
|
28 |
+
from transformers import PretrainedConfig
|
29 |
+
from typing import List
|
30 |
+
from .configuration_TransHLA_II import TransHLA_II_Config
|
31 |
+
|
32 |
+
from transformers import PreTrainedModel
|
33 |
+
|
34 |
+
|
35 |
+
|
36 |
+
class TransHLA_II(nn.Module):
|
37 |
+
def __init__(self,config):
|
38 |
+
super(TransHLA_II, self).__init__()
|
39 |
+
|
40 |
+
max_len = config.max_len
|
41 |
+
n_layers = config.n_layers
|
42 |
+
n_head = config.n_head
|
43 |
+
d_model = config.d_model
|
44 |
+
d_ff = config.d_ff
|
45 |
+
cnn_padding_index = config.cnn_padding_index
|
46 |
+
cnn_num_channel = config.cnn_num_channel
|
47 |
+
region_embedding_size = config.region_embedding_size
|
48 |
+
cnn_kernel_size = config.cnn_kernel_size
|
49 |
+
cnn_padding_size = config.cnn_padding_size
|
50 |
+
cnn_stride = config.cnn_stride
|
51 |
+
pooling_size = config.pooling_size
|
52 |
+
|
53 |
+
self.esm, self.alphabet = esm.pretrained.esm2_t33_650M_UR50D()
|
54 |
+
self.region_cnn1 = nn.Conv1d(
|
55 |
+
d_model, cnn_num_channel, region_embedding_size)
|
56 |
+
self.region_cnn2 = nn.Conv1d(
|
57 |
+
max_len, cnn_num_channel, region_embedding_size)
|
58 |
+
self.padding1 = nn.ConstantPad1d((1, 1), 0)
|
59 |
+
self.padding2 = nn.ConstantPad1d((0, 1), 0)
|
60 |
+
self.relu = nn.ReLU()
|
61 |
+
self.cnn1 = nn.Conv1d(cnn_num_channel, cnn_num_channel, kernel_size=cnn_kernel_size,
|
62 |
+
padding=cnn_padding_size, stride=cnn_stride)
|
63 |
+
self.cnn2 = nn.Conv1d(cnn_num_channel, cnn_num_channel, kernel_size=cnn_kernel_size,
|
64 |
+
padding=cnn_padding_size, stride=cnn_stride)
|
65 |
+
self.maxpooling = nn.MaxPool1d(kernel_size=pooling_size)
|
66 |
+
self.transformer_layers = nn.TransformerEncoderLayer(
|
67 |
+
d_model=d_model, nhead=n_head, dim_feedforward=d_ff, dropout=0.2)
|
68 |
+
self.transformer_encoder = nn.TransformerEncoder(
|
69 |
+
self.transformer_layers, num_layers=n_layers)
|
70 |
+
self.bn1 = nn.BatchNorm1d(d_model)
|
71 |
+
self.bn2 = nn.BatchNorm1d(cnn_num_channel)
|
72 |
+
self.bn3 = nn.BatchNorm1d(cnn_num_channel)
|
73 |
+
self.fc_task = nn.Sequential(
|
74 |
+
nn.Linear(d_model+2*cnn_num_channel, d_model // 4),
|
75 |
+
nn.Dropout(0.3),
|
76 |
+
nn.ReLU(),
|
77 |
+
nn.Linear(d_model // 4, 64),
|
78 |
+
)
|
79 |
+
self.classifier = nn.Linear(64, 2)
|
80 |
+
|
81 |
+
def cnn_block1(self, x):
|
82 |
+
return self.cnn1(self.relu(x))
|
83 |
+
|
84 |
+
def cnn_block2(self, x):
|
85 |
+
x = self.padding2(x)
|
86 |
+
px = self.maxpooling(x)
|
87 |
+
x = self.relu(px)
|
88 |
+
x = self.cnn1(x)
|
89 |
+
x = self.relu(x)
|
90 |
+
x = self.cnn1(x)
|
91 |
+
x = px + x
|
92 |
+
return x
|
93 |
+
|
94 |
+
def structure_block1(self, x):
|
95 |
+
return self.cnn2(self.relu(x))
|
96 |
+
|
97 |
+
def structure_block2(self, x):
|
98 |
+
x = self.padding2(x)
|
99 |
+
px = self.maxpooling(x)
|
100 |
+
x = self.relu(px)
|
101 |
+
x = self.cnn2(x)
|
102 |
+
x = self.relu(x)
|
103 |
+
x = self.cnn2(x)
|
104 |
+
x = px + x
|
105 |
+
return x
|
106 |
+
|
107 |
+
def forward(self, x_in):
|
108 |
+
with torch.no_grad():
|
109 |
+
results = self.esm(x_in, repr_layers=[33], return_contacts=True)
|
110 |
+
emb = results["representations"][33]
|
111 |
+
structure_emb = results["contacts"]
|
112 |
+
output = self.transformer_encoder(emb)
|
113 |
+
representation = output[:, 0, :]
|
114 |
+
representation = self.bn1(representation)
|
115 |
+
cnn_emb = self.region_cnn1(emb.transpose(1, 2))
|
116 |
+
cnn_emb = self.padding1(cnn_emb)
|
117 |
+
conv = cnn_emb + self.cnn_block1(self.cnn_block1(cnn_emb))
|
118 |
+
while conv.size(-1) >= 2:
|
119 |
+
conv = self.cnn_block2(conv)
|
120 |
+
cnn_out = torch.squeeze(conv, dim=-1)
|
121 |
+
cnn_out = self.bn2(cnn_out)
|
122 |
+
|
123 |
+
structure_emb = self.region_cnn2(structure_emb.transpose(1, 2))
|
124 |
+
structure_emb = self.padding1(structure_emb)
|
125 |
+
structure_conv = structure_emb + \
|
126 |
+
self.structure_block1(self.structure_block1(structure_emb))
|
127 |
+
while structure_conv.size(-1) >= 2:
|
128 |
+
structure_conv = self.structure_block2(structure_conv)
|
129 |
+
structure_cnn_out = torch.squeeze(structure_conv, dim=-1)
|
130 |
+
structure_cnn_out = self.bn3(structure_cnn_out)
|
131 |
+
representation = torch.concat(
|
132 |
+
(representation,cnn_out,structure_cnn_out), dim=1)
|
133 |
+
reduction_feature = self.fc_task(representation)
|
134 |
+
reduction_feature = reduction_feature.view(
|
135 |
+
reduction_feature.size(0), -1)
|
136 |
+
logits_clsf = self.classifier(reduction_feature)
|
137 |
+
logits_clsf = torch.nn.functional.softmax(logits_clsf, dim=1)
|
138 |
+
return logits_clsf, reduction_feature
|
139 |
+
|
140 |
+
|
141 |
+
class TransHLA_II_Model(PreTrainedModel):
|
142 |
+
config_class = TransHLA_II_Config
|
143 |
+
|
144 |
+
def __init__(self, config):
|
145 |
+
super().__init__(config)
|
146 |
+
self.model = TransHLA_II(config)
|
147 |
+
|
148 |
+
def forward(self, tensor):
|
149 |
+
return self.model(tensor)
|
pytorch_model.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:c856cdd5edda82995721331e3c1210e78826c4ee80520659bf74b5cf3f37afe9
|
3 |
+
size 2800790141
|