Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<choose_model_class>
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
import datetime
import random
import os
from sklearn.metrics import roc_auc_score
from sklearn.metrics import f1_score
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<define_search_model>
config = {
'folds':15, 'lr': 0.00028, 'seed': 42,
'batch size': 512,
'inf_in':7, 'inf_out':32,
'g_in': 256, 'env2': 256, 'env3': 512,
'g_in': 772, 'g_h1': 256, 'g_h2': 256, 'g_out': 555, 'g_dropout':0.3015,
'c_in': 100, 'c_h1': 256, 'c_h2': 256, 'c_out': 543, 'c_dropout':0.3015,
'merge_out':2048, 'classes_h1': 4096, 'classes_h2': 512, 'classes':206, 'dropout':0.3015,
}
def model_lstm_atten(embedding_matrix):
inp1 = Input(shape=(maxlen,), name='inp1')
 inp2 = Input(shape=(nb_features,), name='inp2')
 x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp1)
 x = SpatialDropout1D(0.25 )(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = concatenate([x, inp2])
 x = Dense(128, activation="relu" )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[inp1, inp2], outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<predict_on_test>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
seed_everything(config['seed'] )
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit({'inp1': train_X, 'inp2': features}, train_y, batch_size=512, epochs=1,
validation_data=({'inp1': val_X, 'inp2': val_features}, val_y))
pred_val_y = model.predict({'inp1': val_X, 'inp2': val_features}, batch_size=1024, verbose=0)
 pred_test_y = model.predict({'inp1': test_X, 'inp2': test_features}, batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y<split>
train_features = pd.DataFrame(pd.read_csv('.. /input/lish-moa/train_features.csv'))
train_targets_nonscored = pd.DataFrame(pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv'))
train_targets_scored = pd.DataFrame(pd.read_csv('.. /input/lish-moa/train_targets_scored.csv'))
test_features = pd.DataFrame(pd.read_csv('.. /input/lish-moa/test_features.csv'))
train_X, val_X, test_X, train_y, val_y, features, val_features, test_features, word_index = load_and_prec()<save_model>
cp_type={'trt_cp':np.array([1,0]), 'ctl_vehicle':np.array([0,1])}
cp_time={24:np.array([0,0,1]), 48:np.array([0,1,0]), 72:np.array([1,0,0])}
cp_dose={'D1':np.array([1,0]), 'D2':np.array([0,1])}
class Dataset:
def __init__(self, x, y):
self.x = x
self.y = y.set_index('sig_id')
 def __getitem__(self, item):
row = self.x.iloc[item].values
y = self.y.loc[row[0]]
xg = np.float32(row[4:4+772])
 xc = np.float32(row[4+772:])
 X = np.float32(np.concatenate([cp_type[row[1]], cp_time[row[2]], cp_dose[row[3]], xg, xc]))
Y = np.float32(y.values)
 return X, Y
def __len__(self):
return len(self.x)
 class DatasetTest:
def __init__(self, x):
self.x = x
def __getitem__(self, item):
row = self.x.iloc[item].values
X = np.float32(np.concatenate([cp_type[row[1]], cp_time[row[2]], cp_dose[row[3]], row[4:]]))
id_ = row[0]
return X, id_
def __len__(self):
return len(self.x )
np.save("train_X",train_X)
 np.save("val_X",val_X)
 np.save("test_X",test_X)
 np.save("train_y",train_y)
 np.save("val_y",val_y)
 np.save("features",features)
 np.save("val_features",val_features)
 np.save("test_features",test_features)
 np.save("word_index.npy",word_index)
 train_X = np.load("train_X.npy")
 val_X = np.load("val_X.npy")
 test_X = np.load("test_X.npy")
 train_y = np.load("train_y.npy")
 val_y = np.load("val_y.npy")
 features = np.load("features.npy")
 val_features = np.load("val_features.npy")
 test_features = np.load("test_features.npy")
 word_index = np.load("word_index.npy" ).item()<load_pretrained>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1), self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
def init_layer(layer):
nn.init.xavier_uniform_(layer.weight)
 if hasattr(layer, "bias"):
if layer.bias is not None:
layer.bias.data.fill_(0.)
def init_bn(bn):
bn.bias.data.fill_(0.)
bn.weight.data.fill_(1.0)
 class Mish(nn.Module):
def __init__(self):
super().__init__()
def forward(self, x):
x = x *(torch.tanh(F.softplus(x)))
 return x
class SigmoidSoftplus(nn.Module):
def __init__(self):
super().__init__()
def forward(self, x):
x = torch.sigmoid(x*F.softplus(x))
return x
class FC(nn.Module):
def __init__(self, in_features, h1, h2, out_features, drop, layer_end=True):
super().__init__()
self.l1 = nn.Linear(in_features, h1)
 self.bn1 = nn.BatchNorm1d(h1)
 self.act1 = nn.LogSigmoid()
self.dr1 = nn.Dropout(drop)
 self.l2 = nn.Linear(h1, h2)
 if layer_end == True:
self.bn2 = nn.BatchNorm1d(h2)
 self.act2 = nn.LogSigmoid()
self.dr2 = nn.Dropout(drop)
 self.l3 = nn.utils.weight_norm(nn.Linear(h2, out_features))
else:
self.l3 = None
init_layer(self.l1)
 init_layer(self.l2)
 if layer_end:
init_layer(self.l3)
 init_bn(self.bn1)
 if layer_end:
init_bn(self.bn2)
 def forward(self, x):
x = self.l1(x)
 x = self.bn1(x)
 x = self.act1(x)
 x = self.dr1(x)
 x = self.l2(x)
 if self.l3 is not None:
x = self.bn2(x)
 x = self.act2(x)
 x = self.dr2(x)
 x = self.l3(x)
 return x
class ENV(nn.Module):
def __init__(self, config):
super().__init__()
self.bn_inf = nn.BatchNorm1d(config['inf_in'])
 self.inf = nn.Linear(config['inf_in'], config['inf_out'])
 self.g = FC(config['g_in'], config['g_h1'], config['g_h2'], config['g_out'], config['g_dropout'], True)
 self.c = FC(config['c_in'], config['c_h1'], config['c_h2'], config['c_out'], config['c_dropout'], True)
 self.merge = nn.utils.weight_norm(nn.Linear(config['g_out'] + config['c_out'] + 32, config['merge_out']))
init_bn(self.bn_inf)
 init_layer(self.inf)
 def forward(self, x):
x_0 = x
x1 = x[:, :7]
x2 = x[:, 7:7+772]
x3 = x[:, 7+772:]
inf = self.inf(self.bn_inf(x1))
g = self.g(x2)
 c = self.c(x3)
 x = torch.cat([g, c, inf], 1)
 x = self.merge(x)
 return x
class MOAModel(nn.Module):
def __init__(self, config):
super().__init__()
self.env = ENV(config)
 self.act1 = SigmoidSoftplus()
self.fc = FC(config['merge_out'], config['classes_h1'], config['classes_h2'], config['classes'], config['dropout'], False)
 self.bn2 = nn.BatchNorm1d(config['classes_h2'])
 self.act2 = nn.LogSigmoid()
self.dr2 = nn.Dropout(config['dropout'])
 self.l2 = nn.utils.weight_norm(nn.Linear(config['classes_h2'], config['classes']))
init_bn(self.bn2)
 init_layer(self.l2)
 def forward(self, x):
x = self.env(x)
 x = self.act1(x)
 b = x.size(0)
 x = x.reshape(b, -1)
 x = self.fc(x)
 x = self.bn2(x)
 x = self.act2(x)
 x = self.dr2(x)
 x = self.l2(x)
 return x
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_fasttext(word_index)
 embedding_matrix_3 = load_para(word_index )<compute_test_metric>
dataset_train = train_features[scored['fold'] != config['folds'] - 1].reset_index(drop=True)
 dataset_valid = train_features[scored['fold'] == config['folds'] - 1].reset_index(drop=True)
 print(len(dataset_train), len(dataset_valid))
embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2, embedding_matrix_3], axis = 0)
 np.shape(embedding_matrix )<define_variables>
BATCH_SIZE = config["batch size"]
dts_train = Dataset(dataset_train, train_targets_scored)
 dts_valid = Dataset(dataset_valid, train_targets_scored)
 dts_test = DatasetTest(test_features)
 train_loader = torch.utils.data.DataLoader(dts_train,
batch_size=BATCH_SIZE,
shuffle=True,
sampler=None,
num_workers=4)
 valid_loader = torch.utils.data.DataLoader(dts_valid,
batch_size=BATCH_SIZE,
shuffle=False,
sampler=None,
num_workers=4)
 print(len(train_loader), len(valid_loader))
outputs = []<predict_on_test>
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 model = MOAModel(config)
 model = model.to(device )
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix), epochs = EPOCHS)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention'] )<compute_test_metric>
criteric1 = SmoothBCEwLogits(smoothing = 0.001)
 criteric2 = nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(model.parameters() , lr=config['lr'])
 scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1, last_epoch=-1 )
score = 0
thresh =.5
for i in np.arange(0.1, 0.991, 0.01):
y_val =(np.array(pred_val_y)> i ).astype(np.int)
 temp_score = f1_score(val_y, y_val)
 if(temp_score > score):
score = temp_score
thresh = i
print("CV: {}, Threshold: {}".format(score, thresh))<save_to_csv>
class GaussianNoise(nn.Module):

 def __init__(self, sigma=0.1, is_relative_detach=True):
super().__init__()
self.sigma = sigma
self.is_relative_detach = is_relative_detach
self.noise = torch.tensor(0 ).to(device)
 def forward(self, x):
if self.training and self.sigma != 0:
scale = self.sigma * x.detach() if self.is_relative_detach else self.sigma * x
sampled_noise = self.noise.repeat(*x.size() ).float().normal_() * scale
x = x + sampled_noise
return x
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > thresh ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<train_model>
def mixup_data(x, y, alpha=1.0, use_cuda=True):
if alpha > 0.:
lam = np.random.beta(alpha, alpha)
 else:
lam = 1.
 batch_size = x.size() [0]
if use_cuda:
index = torch.randperm(batch_size ).cuda()
else:
index = torch.randperm(batch_size)
 mixed_x = lam * x +(1 - lam)* x[index,:]
y_a, y_b = y, y[index]
return mixed_x, y_a, y_b, lam
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )<load_from_csv>
best_loss = np.inf
epoch_start = 1
model_name_best = ''
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 _train = add_features(train_df)
 _test = add_features(test_df)
 features = _train.fillna(0)
 test_features = _test.fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features)
<import_modules>
print(datetime.datetime.now())
 EARLY_STOPPING_STEPS = 20
EARLY_STOP = True
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
noise = GaussianNoise(0.33)
 history = []
for epoch in range(epoch_start, 15000+1):
model.train()
loss_total = 0.0
count = 0
for b_idx, xy in enumerate(train_loader):
X = torch.FloatTensor(xy[0])
 Y = torch.FloatTensor(xy[1])
 X = X.to(device)
 Y = Y.to(device)
 for param in model.parameters() :
param.grad = None
k = np.random.randint(2)
 if k == 0:
pred = model(X)
 loss = criteric1(pred, Y)
 if k == 1:
mixed_x, y_a, y_b, lam = mixup_data(X, Y, 0.4)
 pred = model(mixed_x)
 loss = lam * criteric1(pred, y_a)+(1 - lam)* criteric1(pred, y_b)
 if k == 2:
X = noise(X)
 pred = model(X)
 loss = criteric1(pred, Y)
 if k == 3:
mixed_x, y_a, y_b, lam = mixup_data(X, Y, 0.4)
 mixed_x = noise(mixed_x)
 pred = model(mixed_x)
 loss = lam * criteric1(pred, y_a)+(1 - lam)* criteric1(pred, y_b)
 loss.backward()
if b_idx % 1 == 0:
optimizer.step()
scheduler.step(epoch + b_idx / len(train_loader))
pred = torch.sigmoid(pred)
 loss_total += loss.item()
count += 1
loss_toat_mean = loss_total/count
print(f"{datetime.datetime.now() }: {epoch}.Train | loss={(loss_toat_mean):.6f}")
 model.eval()
loss_total = 0.0
count = 0
for b_idx, xy in enumerate(valid_loader):
X = torch.FloatTensor(xy[0])
 Y = torch.FloatTensor(xy[1])
 X = X.to(device)
 Y = Y.to(device)
 with torch.no_grad() :
pred = model(X)
 loss = criteric2(pred, Y)
 loss_total += loss.item()
count += 1
pred = torch.sigmoid(pred)
 loss_toat_mean = loss_total/count
print(f"{datetime.datetime.now() }: {epoch}.Valid | loss={(loss_toat_mean):.6f}")
 if best_loss >= loss_total/count :
print(f"\tSave: {best_loss} -> {(loss_total/count)}")
 best_loss = loss_total/count
if os.path.isfile(model_name_best):
os.remove(model_name_best)
 model_name_best = f'{best_loss}_model_base.pt'
torch.save(model.state_dict() , model_name_best)
 early_step = 0
elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
epoch_start += 1
import re
import numpy as np
import pandas as pd
from tqdm import tqdm
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
from gensim.models import KeyedVectors
from sklearn.preprocessing import StandardScaler
import keras
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, Embedding, Dropout, Reshape, Flatten, LSTM, Bidirectional
from keras.layers.merge import concatenate
from keras.models import Model
from keras.layers.normalization import BatchNormalization
from keras.callbacks import EarlyStopping, ModelCheckpoint<string_transform>
msd = torch.load(f'{best_loss}_model_base.pt', map_location=device)
 model.load_state_dict(msd)
 model = model.to(device )
def text_to_wordlist(text, remove_stopwords=False, stem_words=False):
text = text.lower().split()
if remove_stopwords:
stops = set(stopwords.words("english"))
text = [w for w in text if not w in stops]
text = " ".join(text)
 text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
 text = re.sub(r"what's", "what is ", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"can't", "cannot ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r",", " ", text)
 text = re.sub(r"\.", " ", text)
 text = re.sub(r"!", " ! ", text)
 text = re.sub(r"\/", " ", text)
 text = re.sub(r"\^", " ^ ", text)
 text = re.sub(r"\+", " + ", text)
 text = re.sub(r"\-", " - ", text)
 text = re.sub(r"\=", " = ", text)
 text = re.sub(r"'", " ", text)
 text = re.sub(r"(\d+ )(k)", r"\g<1>000", text)
 text = re.sub(r":", " : ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r" u s ", " american ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e - mail", "email", text)
 text = re.sub(r"j k", "jk", text)
 text = re.sub(r"\s{2,}", " ", text)
 if stem_words:
text = text.split()
stemmer = SnowballStemmer('english')
 stemmed_words = [stemmer.stem(word)for word in text]
text = " ".join(stemmed_words)
 return(text)
 def read_train_data(file):
texts = []
labels = []
df_train = pd.read_csv(file)
 line_num = 0
for idx in range(len(df_train)) :
texts.append(text_to_wordlist(df_train['question_text'][idx]))
labels.append(df_train['target'][idx])
 line_num += 1
return texts, labels
def read_test_data(file):
texts = []
ids = []
df_test = pd.read_csv(file)
 line_num = 0
for idx in range(len(df_test)) :
texts.append(text_to_wordlist(df_test['question_text'][idx]))
ids.append(df_test['qid'][idx])
 line_num += 1
return texts, ids
def preprocess_data(train_data_file, test_data_file, max_seq_len, split_ratio):
texts, labels= read_train_data(train_data_file)
 print('Finished loading train.csv: %s samples' % len(texts))
test_texts, test_ids = read_test_data(test_data_file)
 print('Finished loading test.csv: %s samples' % len(test_texts))
tokenizer = Tokenizer(num_words=200000)
 tokenizer.fit_on_texts(texts + test_texts)
 word_index = tokenizer.word_index
print('%s tokens in total' % len(word_index))
train_sequences = tokenizer.texts_to_sequences(texts)
 test_sequences = tokenizer.texts_to_sequences(test_texts)
 x = pad_sequences(train_sequences, maxlen=max_seq_len, padding='post', truncating='post')
 test_x = pad_sequences(test_sequences, maxlen=max_seq_len, padding='post', truncating='post')
 num_samples = len(x)
 perm = np.random.permutation(num_samples)
 idx = int(num_samples*split_ratio)
 idx_train = perm[:idx]
idx_val = perm[idx:]
train_x = x[idx_train]
val_x = x[idx_val]
y = np.array(labels)
 train_y = y[idx_train]
val_y = y[idx_val]
print('Shape of training data: {}'.format(train_x.shape))
print('Shape of training label: {}'.format(train_y.shape))
print('Shape of val data: {}'.format(val_x.shape))
print('Shape of val label: {}'.format(val_y.shape))
print('Shape of test data: {}'.format(test_x.shape))
return train_x, train_y, val_x, val_y, test_x, test_ids, word_index
def load_embeddings_index(file, embedding_dim):
embeddings_index = {}
f=open(file)
 for line in tqdm(f):
values = line.split(" ")
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
print('Found %s word vectors.' % len(embeddings_index))
return embeddings_index
def generate_embedding_layer(word_index, embeddings_index):
nb_words = len(word_index)+ 1
embeddings_matrix = np.zeros(( nb_words, embedding_dim))
word_out_network = []
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embeddings_matrix[i] = embedding_vector
else:
word_out_network.append(word)
 percent = round(100*len(word_out_network)/len(word_index),1)
 print('%s percent of words out of network' % percent)
 return embeddings_matrix
def build_model(max_seq_len, word_index, embedding_dim, embedding_matrix):
inp = Input(shape=(max_seq_len,), dtype='int32')
 x = Embedding(len(word_index)+1,
embedding_dim,
input_length=max_seq_len,
weights=[embedding_matrix],
trainable=False )(inp)
 x = LSTM(64,dropout=0.2, recurrent_dropout=0.2 )(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(32, activation='relu' )(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 preds = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=inp, outputs=preds)
 return model<load_from_csv>
BATCH_SIZE = 1
test_features = pd.DataFrame(pd.read_csv('.. /input/lish-moa/test_features.csv'))
test_loading = torch.utils.data.DataLoader(DatasetTest(test_features),
batch_size=BATCH_SIZE,
shuffle=False,
sampler=None,
num_workers=4 )
max_seq_len = 30
split_ratio = 0.8
train_file = '.. /input/quora-insincere-questions-classification/train.csv'
test_file = '.. /input/quora-insincere-questions-classification/test.csv'
train_x, train_y, val_x, val_y, test_x, test_ids, word_index = \
preprocess_data(train_file, test_file, max_seq_len, split_ratio )<categorify>
train_targets_scored = pd.DataFrame(pd.read_csv('.. /input/lish-moa/train_targets_scored.csv'))
train_targets_scored
embedding_dim = 300
embedding_matrix = np.zeros(( max(list(word_index.values())) + 1, embedding_dim), dtype = 'float32')
 embedding_file='.. /input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'
f = open(embedding_file)
 for line in tqdm(f):
values = line.split(" ")
 word = values[0]
if word not in word_index:
continue
embedding_matrix[word_index[word]] = np.asarray(values[1:], dtype='float32')
 f.close() 
<choose_model_class>
test_features = pd.DataFrame(pd.read_csv('.. /input/lish-moa/test_features.csv'))
sample_submission = pd.DataFrame(pd.read_csv('.. /input/lish-moa/sample_submission.csv'))
model = build_model(max_seq_len, word_index, embedding_dim, embedding_matrix)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['acc'])
 model.summary()<train_model>
print(datetime.datetime.now())
 res = []
model.eval()
for b_idx, xy in enumerate(test_loading):
X = torch.FloatTensor(xy[0])
 X = X.to(device)
 with torch.no_grad() :
pred = model(X)
 pred = torch.sigmoid(pred)
 for i in range(len(pred)) :
pr = pred[i].cpu().data.numpy()
pr = np.clip(pr,0.0005,0.999)
 res.append([xy[1][i]] + pr.tolist())
 print(len(res))
nb_epoches = 200
early_stopping = EarlyStopping(monitor='val_loss', patience=5)
 model_name = 'model_best.h5'
model_checkpoint = ModelCheckpoint(model_name, save_best_only=True)
 hist = model.fit(train_x, train_y, \
validation_data=(val_x, val_y), \
epochs=nb_epoches, batch_size=2048, shuffle=True, verbose=2, \
callbacks=[early_stopping, model_checkpoint])
 model.load_weights(model_name)
 best_val_score = min(hist.history['val_loss'])
 print('Min val loss is', best_val_score )<save_to_csv>
res_test = pd.DataFrame(res, columns=train_targets_scored.columns)
 res_test
preds = model.predict(test_x, batch_size=1024, verbose=1)
 preds =(preds > 0.35 ).astype(int)
 sub = pd.DataFrame({'qid':test_ids, 'prediction':preds.ravel() })
 sub.to_csv('submission.csv', index=False )<define_variables>
target_cols = train_targets_scored.drop('sig_id', axis=1 ).columns.values.tolist()
<load_from_csv><EOS> 
sub = sample_submission.drop(columns=target_cols ).merge(res_test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )