Kaggle competition: VSB Power Line Fault Detection
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: vsb-power-line-fault-detection<categorify>
import pandas as pd
import pyarrow.parquet as pq
import os
import numpy as np
from joblib import Parallel, delayed
import tensorflow as tf
from tensorflow import set_random_seed
from keras.layers import *
from keras.models import Model
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from sklearn.metrics import matthews_corrcoef
from keras import backend as K
from keras import optimizers
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from keras.callbacks import *
outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arrays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
<normalization>
N_SPLITS = 5
sample_size = 800000
MAX_THREADS = 2
RANDOM_SEED = 2019
inputs_count = build_count_features(df_train ).apply(np.log1p)
 test_inputs_count = build_count_features(df_test ).apply(np.log1p)
 ss = StandardScaler()
inputs_count = ss.fit_transform(inputs_count)
 test_inputs_count = ss.transform(test_inputs_count)
 inputs_count = np.asarray(inputs_count, dtype=np.float32)
 test_inputs_count = np.asarray(test_inputs_count, dtype=np.float32 )<init_hyperparams>
np.random.seed(RANDOM_SEED)
 set_random_seed(RANDOM_SEED )
n_splits = 5
epochs = 10
batch_size = 4
learning_rate = 3e-5
gkf = GroupKFold(n_splits=n_splits ).split(X=df_train.question_body, groups=df_train.question_body)
 test_inputs += [test_inputs_count]
rhos_val = []
test_preds = []
model_10inp = create_model_10inputs()
model_6inp = create_model_6inputs()
for fold,(train_idx, valid_idx)in enumerate(gkf):
train_inputs = [inputs[i][train_idx] for i in range(len(inputs)) ]
train_inputs += [inputs_count[train_idx]]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(len(inputs)) ]
valid_inputs += [inputs_count[valid_idx]]
valid_outputs = outputs[valid_idx]
K.clear_session()
if False:
model = create_model()
optimizer = tf.keras.optimizers.Adam(learning_rate)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 custom_callback = CustomCallback(
 valid_data=(valid_inputs, valid_outputs),
batch_size=batch_size, 
)
 model.fit(train_inputs, train_outputs,
epochs=epochs,
batch_size=batch_size,
callbacks=[custom_callback])
 model.save_weights(f'bert-base-{fold}of{n_splits}-{epochs}epoch.h5')
 test_preds.append(model.predict(test_inputs))
if True:
weights_10 = WEIGHTS_FILES_10[fold]
print(f'
Load weights from {weights_10}')
 model_10inp.load_weights(weights_10)
 test_preds.append(model_10inp.predict(test_inputs, batch_size=batch_size))
weights_6 = WEIGHTS_FILES_6[fold]
print(f'
Load weights from {weights_6}')
 model_6inp.load_weights(weights_6)
 test_preds.append(model_6inp.predict(test_inputs[:6], batch_size=batch_size))
weights_6_noes = WEIGHTS_FILES_6_NOES[fold]
print(f'
Load weights from {weights_6}')
 model_6inp.load_weights(weights_6_noes)
 test_preds.append(model_6inp.predict(test_inputs[:6], batch_size=batch_size))<save_to_csv>
class StatefullMCC(Layer):
def __init__(self, thresholds, **kwargs):
super(StatefullMCC, self ).__init__(**kwargs)
 self.thresholds = thresholds
self.stateful = True
self.name='matthews_correlation'
def reset_states(self):
K.get_session().run(tf.variables_initializer(self.local_variable))
def metric_variable(self, shape, dtype, validate_shape=True, name=None):
return tf.Variable(
 np.zeros(shape),
dtype=dtype,
trainable=False,
collections=[tf.GraphKeys.LOCAL_VARIABLES],
validate_shape=validate_shape,
name=name, 
)
 def initialize_vars(self, y_true, y_pred):
self.tp = [None]*len(self.thresholds)
 self.tn = [None]*len(self.thresholds)
 self.fp = [None]*len(self.thresholds)
 self.fn = [None]*len(self.thresholds)
 for i,_ in enumerate(self.thresholds):
self.tp[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='tp%d'%i)
 self.tn[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='tn%d'%i)
 self.fp[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='fp%d'%i)
 self.fn[i] = self.metric_variable(shape=[1], dtype=tf.int64, validate_shape=False, name='fn%d'%i)
 tp_op = tf.assign_add(self.tp[i], tf.count_nonzero(y_true * y_pred[i], axis=0))
self.add_update(tp_op)
 tn_op = tf.assign_add(self.tn[i], tf.count_nonzero(( 1-y_true)*(1-y_pred[i]), axis=0))
self.add_update(tn_op)
 fp_op = tf.assign_add(self.fp[i], tf.count_nonzero(( 1-y_true)*y_pred[i], axis=0))
self.add_update(fp_op)
 fn_op = tf.assign_add(self.fn[i], tf.count_nonzero(y_true*(1-y_pred[i]), axis=0))
self.add_update(fn_op)
 self.local_variable = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES)
 def __call__(self, y_true, y_pred):
rounded_preds = [K.cast(K.greater_equal(y_pred, t), 'float32')for t in self.thresholds]
self.initialize_vars(y_true, rounded_preds)
 mcc_vec = []
for i,_ in enumerate(self.thresholds):
num = tf.cast(( self.tp[i] * self.tn[i] - self.fp[i] * self.fn[i]), 'float32')
 den = K.sqrt(tf.cast(( self.tp[i] + self.fp[i])*(self.tp[i] + self.fn[i])*(self.tn[i] + self.fp[i])*(self.tn[i] + self.fn[i]), 'float32'))
mcc_vec.append(num/(den + tf.constant(K.epsilon())))
 return K.max(mcc_vec )
df_sub.iloc[:, 1:] = np.average(test_preds, axis=0)
 df_sub.to_csv('submission.csv', index=False )<import_modules>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.glorot_uniform(RANDOM_SEED)
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
import numpy as np
import pandas as pd
import os 
<install_modules>
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()
!pip install.. /input/sacremoses/sacremoses-master/
!pip install.. /input/transformers/transformers-master/<import_modules>
def get_features(dataset='train', split_parts=10):
if dataset == 'train':
cache_file = 'X.npy'
meta_file = '.. /input/metadata_train.csv'
elif dataset == 'test':
cache_file = 'X_test.npy'
meta_file = '.. /input/metadata_test.csv'
if os.path.isfile(cache_file):
X = np.load(cache_file)
 y = None
if dataset == 'train':
y = np.load('y.npy')
 else:
meta_df = pd.read_csv(meta_file)
 data_measurements = meta_df.pivot(index='id_measurement', columns='phase', values='signal_id')
 data_measurements = data_measurements.values
data_measurements = np.array_split(data_measurements, split_parts, axis=0)
 X = Parallel(n_jobs=min(split_parts, MAX_THREADS), verbose=1 )(delayed(prep_data )(p, dataset)for p in data_measurements)
 try:
y = meta_df.loc[meta_df['phase']==0, 'target'].values
except:
y = None
X = np.concatenate(X, axis=0)
 if dataset == 'train':
np.save("X.npy",X)
 np.save("y.npy",y)
 elif dataset == 'test':
np.save("X_test.npy",X)
 return X, y
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from tqdm.notebook import tqdm
import tensorflow_hub as hub
import tensorflow as tf
import tensorflow.keras.backend as K
import gc
import time
import random
import os
import torch
from scipy.stats import spearmanr
from sklearn.model_selection import KFold,StratifiedKFold
from math import floor, ceil
from transformers import AdamW,BertForSequenceClassification, get_cosine_with_hard_restarts_schedule_with_warmup 
<load_from_csv>
max_num = 127
min_num = -128
sample_submission = pd.read_csv(".. /input/google-quest-challenge/sample_submission.csv")
 test = pd.read_csv(".. /input/google-quest-challenge/test.csv")
 train = pd.read_csv(".. /input/google-quest-challenge/train.csv")
 MAX_SEQUENCE_LENGTH = 512<categorify>
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 if t_len > t_new_len:
ind1 = floor(t_new_len/2)
 ind2 = ceil(t_new_len/2)
 t = t[:ind1]+t[-ind2:]
else:
t = t[:t_new_len]
if q_len > q_new_len:
ind1 = floor(q_new_len/2)
 ind2 = ceil(q_new_len/2)
 q = q[:ind1]+q[-ind2:]
else:
q = q[:q_new_len]
if a_len > a_new_len:
ind1 = floor(a_new_len/2)
 ind2 = ceil(a_new_len/2)
 a = a[:ind1]+a[-ind2:]
else:
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<set_options>
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<define_variables>
def prep_data(signal_ids, dataset="train"):
signal_ids_all = np.concatenate(signal_ids)
 if dataset == "train":
praq_data = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in signal_ids_all] ).to_pandas()
elif dataset == "test":
praq_data = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in signal_ids_all] ).to_pandas()
else:
raise ValueError("Unknown dataset")
 X = []
for sids in signal_ids:
data = praq_data[[str(s)for s in sids]].values.T
X_signal = [transform_ts(signal)for signal in data]
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 return X
class TextDataset(torch.utils.data.TensorDataset):
def __init__(self, x_train, idxs, targets=None):
self.input_ids = x_train[0][idxs]
self.input_masks = x_train[1][idxs]
self.input_segments = x_train[2][idxs]
self.targets = targets[idxs] if targets is not None else np.zeros(( x_train[0].shape[0], 30))
def __getitem__(self, idx):
input_ids = self.input_ids[idx]
input_masks = self.input_masks[idx]
input_segments = self.input_segments[idx]
target = self.targets[idx]
return input_ids, input_masks, input_segments, target
def __len__(self):
return len(self.input_ids )<import_modules>
X, y = get_features("train", split_parts=6 )
from transformers import BertTokenizer,BertConfig<load_pretrained>
def model_lstm(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
init_glorot_uniform = initializers.glorot_uniform(seed=RANDOM_SEED)
 init_orthogonal = initializers.orthogonal(seed=RANDOM_SEED)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True, kernel_initializer=init_glorot_uniform, recurrent_initializer=init_orthogonal))(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True, kernel_initializer=init_glorot_uniform, recurrent_initializer=init_orthogonal))(x)
 x = Attention(input_shape[1] )(x)
 x = Dense(64, activation="relu", kernel_initializer=init_glorot_uniform )(x)
 x = Dense(1, activation="sigmoid", kernel_initializer=init_glorot_uniform )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[StatefullMCC(np.linspace(0.45,0.55,11)) ])
 return model
tokenizer = BertTokenizer.from_pretrained('.. /input/bertbaseuncased/bert-base-uncased-vocab.txt' )<prepare_x_and_y>
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=RANDOM_SEED ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = model_lstm(train_X.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit(train_X, train_y, batch_size=128, epochs=50, validation_data=[val_X, val_y], callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict(val_X, batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape
x_train = compute_input_arays(train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 y_train = compute_output_arrays(train, output_categories)
 x_test = compute_input_arays(test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )<load_pretrained>
def threshold_search(y_true, y_proba):
thresholds = np.linspace(0.0,1.0,101)
 scores = [matthews_corrcoef(y_true,(y_proba > t ).astype(np.uint8)) for t in thresholds]
best_idx = np.argmax(scores)
 return thresholds[best_idx], scores[best_idx]
bert_config = BertConfig.from_pretrained('.. /input/bertbaseuncased/bert-base-uncased-config.json')
 bert_config.num_labels = 30<import_modules>
best_threshold, best_score = threshold_search(y_val, preds_val)
 print(best_threshold, best_score )
class CustomizedBert(BertPreTrainedModel):
r
def __init__(self, config):
super(CustomizedBert, self ).__init__(config)
 self.num_labels = config.num_labels
self.bert = BertModel(config)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.classifier = nn.Linear(config.hidden_size*2, self.config.num_labels)
 self.init_weights()
def forward(
 self,
input_ids=None,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
inputs_embeds=None,
labels=None, 
):
outputs = self.bert(
 input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 avg_pool = torch.mean(outputs[0], 1)
 max_pool, _ = torch.max(outputs[0], 1)
 pooled_output = torch.cat(( max_pool, avg_pool), 1)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 outputs =(logits,)+ outputs[2:]
if labels is not None:
if self.num_labels == 1:
loss_fct = MSELoss()
loss = loss_fct(logits.view(-1), labels.view(-1))
else:
loss_fct = CrossEntropyLoss()
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
outputs =(loss,)+ outputs
return outputs<find_best_params>
%%time
meta_test = pd.read_csv('.. /input/metadata_test.csv' )
class callback:
def __init__(self):
self.score = list()
self.model = list()
self.data = list()
def put(self, model,data, score):
self.score.append(score)
 self.model.append(model)
 self.data.append(data)
 def get_model(self):
ind = np.argmin(self.score)
 return self.model[ind]
def get_data(self):
ind = np.argmin(self.score)
 return self.data[ind]<prepare_x_and_y>
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()
NFOLDS = 5
BATCH_SIZE = 4
EPOCHS = 4
SEED = 8516
num_warmup_steps = 100
lr = 5e-5
gradient_accumulation_steps = 1
seed_everything(SEED)
 model_list = list()
y_oof = np.zeros(( len(train), 30))
test_pred = np.zeros(( len(test), 30))
kf = KFold(n_splits=NFOLDS, shuffle=True)
 test_loader = torch.utils.data.DataLoader(TextDataset(x_test, test.index),batch_size=BATCH_SIZE, shuffle=False)
 for i,(train_idx, valid_idx)in enumerate(kf.split(x_train[0])) :
print(f'fold {i+1}')
 gc.collect()
train_loader = torch.utils.data.DataLoader(TextDataset(x_train, train_idx, y_train),batch_size=BATCH_SIZE, shuffle=True)
 val_loader = torch.utils.data.DataLoader(TextDataset(x_train, valid_idx, y_train),batch_size=BATCH_SIZE, shuffle=False)
 t_total = len(train_loader)//gradient_accumulation_steps*EPOCHS
net = CustomizedBert.from_pretrained('.. /input/bertbaseuncased/', config=bert_config)
 net.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss()
optimizer = AdamW(net.parameters() , lr = lr)
 scheduler = get_cosine_with_hard_restarts_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=t_total)
 cb = callback()
for epoch in range(EPOCHS):
start_time = time.time()
avg_loss = 0.0
net.train()
for step, data in enumerate(train_loader):
input_ids, input_masks, input_segments, labels = data
pred = net(input_ids = input_ids.long().cuda() ,
labels = None,
attention_mask = input_masks.cuda() ,
token_type_ids = input_segments.long().cuda() 
)[0]
loss = loss_fn(pred, labels.cuda())
 avg_loss += loss.item()
loss = loss / gradient_accumulation_steps
loss.backward()
if(step + 1)% gradient_accumulation_steps == 0:
optimizer.step()
scheduler.step()
optimizer.zero_grad()
avg_val_loss = 0.0
valid_preds = np.zeros(( len(valid_idx), 30))
true_label = np.zeros(( len(valid_idx), 30))
net.eval()
for j,data in enumerate(val_loader):
input_ids, input_masks, input_segments, labels = data
pred = net(input_ids = input_ids.long().cuda() ,
labels = None,
attention_mask = input_masks.cuda() ,
token_type_ids = input_segments.long().cuda() 
)[0]
loss_val = loss_fn(pred, labels.cuda())
 avg_val_loss += loss_val.item()
valid_preds[j * BATCH_SIZE:(j+1)* BATCH_SIZE] = torch.sigmoid(pred ).cpu().detach().numpy()
true_label[j * BATCH_SIZE:(j+1)* BATCH_SIZE] = labels
elapsed_time = time.time() - start_time
score = 0
for i in range(30):
s = np.nan_to_num(
 spearmanr(true_label[:, i], valid_preds[:, i] ).correlation / 30)
 score += s
print('Epoch {}/{} \t loss={:.4f}\t val_loss={:.4f}\t spearmanr={:.4f}\t time={:.2f}s'.format(epoch+1, EPOCHS, avg_loss/len(train_loader),avg_val_loss/len(val_loader),score, elapsed_time))
cb.put(net,valid_preds,avg_val_loss/len(val_loader))
model_list.append(cb.get_model())
 y_oof[valid_idx] = cb.get_data()
result = list()
net.eval()
with torch.no_grad() :
for data in test_loader:
input_ids, input_masks, input_segments, labels = data
y_pred = net(input_ids = input_ids.long().cuda() ,
labels = None,
attention_mask = input_masks.cuda() ,
token_type_ids = input_segments.long().cuda() , 
)[0]
result.extend(torch.sigmoid(y_pred ).cpu().detach().numpy())
 test_pred += np.array(result)/NFOLDS 
<save_to_csv>
%%time
X_test_input, _ = get_features("test" )
sample_submission.loc[:, output_categories] = test_pred
sample_submission.to_csv('submission.csv', index=False )<install_modules>
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()
!pip install.. /input/huggingface-transformers/sacremoses-master/sacremoses-master
!pip install.. /input/huggingface-transformers/transformers-master/transformers-master<set_options>
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict(X_test_input, batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3)
tqdm.pandas()
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape
<string_transform><EOS> 
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()