Kaggle competition: VSB Power Line Fault Detection
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: vsb-power-line-fault-detection<set_options>
import os
import pandas as pd
import numpy as np
import pyarrow.parquet as pq
from keras.layers import *
from keras.callbacks import *
from keras.initializers import *
from keras import optimizers
from keras import backend as K
from keras.models import Model
import tensorflow as tf
from sklearn.model_selection import StratifiedKFold
from tqdm import tqdm
del bert_base_cased_models, test_loader
torch.cuda.empty_cache()
gc.collect()<load_pretrained>
sample_size = 800000
test_loader, _ = get_test_loader(model_type="bert-base-uncased", batch_size=32 )<predict_on_test>
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()
bert_base_uncased_models = create_bert_base_uncased_models()
bert_base_uncased_preds = predict(bert_base_uncased_models, test_loader )<set_options>
max_num = 127
min_num = -128
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]
del bert_base_uncased_models, test_loader
torch.cuda.empty_cache()
gc.collect()<define_variables>
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )
preds = 0.25 *(0.5 * bert_base_uncased_preds + 0.5 * bert_base_uncased_question_answer_preds)\
+ 0.25 *(0.5 * bert_base_cased_preds + 0.5 * bert_base_cased_question_answer_preds)\
+ 0.25 *(0.5 * xlnet_base_cased_preds + 0.5 * xlnet_base_cased_question_answer_preds)\
+ 0.25 *(0.5 * roberta_base_preds + 0.5 * roberta_base_question_answer_preds)
<save_to_csv>
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y
sub[TARGET_COLUMNS] = bert_base_uncased_preds
sub.to_csv('submission_bert_base_uncased.csv', index=False)
 sub[TARGET_COLUMNS] = bert_base_cased_preds
sub.to_csv('submission_bert_base_cased.csv', index=False)
 sub[TARGET_COLUMNS] = xlnet_base_cased_preds
sub.to_csv('submission_xlnet_base_cased.csv', index=False)
<prepare_output>
X = []
y = []
def load_all() :
total_size = len(df_train)
 for ini, end in [(0, int(total_size/2)) ,(int(total_size/2), total_size)]:
X_temp, y_temp = prep_data(ini, end)
 X.append(X_temp)
 y.append(y_temp)
 load_all()
X = np.concatenate(X)
 y = np.concatenate(y )
sub[TARGET_COLUMNS] = preds<load_from_csv>
def matthews_correlation(y_true, y_pred):
y_pred = tf.convert_to_tensor(y_pred, np.float32)
 y_true = tf.convert_to_tensor(y_true, np.float32)
 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )
test = pd.read_csv(f'{DATA_DIR}/test.csv' )<merge>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
test = test.set_index('qa_id' ).join(sub.set_index('qa_id'))<feature_engineering>
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )
def postprocessing(oof_df):
oof_values = oof_df[TARGET_COLUMNS].values
DEGREE = 90
oof_values = np.around(oof_values * DEGREE)/ DEGREE
oof_df[TARGET_COLUMNS] = oof_values
return oof_df<categorify>
def model_lstm(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
x = Bidirectional(CuDNNLSTM(128, return_sequences=True,
kernel_initializer=glorot_normal(seed=1029),
recurrent_initializer=orthogonal(gain=1.0, seed=1029)) )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True,
kernel_initializer=glorot_normal(seed=1029),
recurrent_initializer=orthogonal(gain=1.0, seed=1029)) )(x)
 x_1 = Attention(input_shape[1] )(x)
 x_1 = Dropout(0.5 )(x_1)
 x_2 = Capsule(num_capsule=8, dim_capsule=8, routings=4, share_weights=True )(x)
 x_2 = Flatten()(x_2)
 x_2 = Dropout(0.5 )(x_2)
 x_rcnn = Conv1D(filters=128,
kernel_size=1,
kernel_initializer='he_uniform' )(inp)
 x_rcnn = Activation('relu' )(x_rcnn)
 x_rcnn_atten = Attention(input_shape[1] )(x_rcnn)
 x_rcnn_capsule = Capsule(num_capsule=8, dim_capsule=8, routings=4, share_weights=True )(x_rcnn)
 x_rcnn_capsule = Flatten()(x_rcnn_capsule)
 conc = concatenate([x_1, x_2, x_rcnn_atten, x_rcnn_capsule])
 conc = Dense(512, activation="relu" )(conc)
 conc = Dropout(0.5 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model
test = postprocessing(test )<count_values>
N_SPLITS = 5
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=1029 ).split(X, y))
preds_val = []
y_val = []
best_scores = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = model_lstm(train_X.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 history = model.fit(train_X, train_y, batch_size=128, epochs=50, validation_data=[val_X, val_y], callbacks=[ckpt])
 best_scores.append(np.max(history.history['val_matthews_correlation']))
model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict(val_X, batch_size=512))
y_val.append(val_y)
 print("
" + str(best_scores))
print("
" + str(np.mean(best_scores)) )
for column in TARGET_COLUMNS:
print(test[column].value_counts() )<prepare_output>
preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 print(preds_val.shape, y_val.shape )
sub = test[TARGET_COLUMNS].reset_index()<feature_engineering>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = K.eval(matthews_correlation(y_true.astype(np.float64),(y_proba > threshold ).astype(np.float64)))
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
return search_result
sub[ sub[TARGET_COLUMNS] > 1.0] = 1.0
sub[ sub[TARGET_COLUMNS] < 0] = 0<count_values>
best_threshold = threshold_search(y_val, preds_val)['threshold']
print(best_threshold )
sub['question_type_spelling'].value_counts()<save_to_csv>
meta_test = pd.read_csv('.. /input/metadata_test.csv')
 meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()
sub.to_csv('submission.csv', index=False )<save_to_csv>
%%time
first_sig = meta_test.index[0]
n_parts = 10
max_line = len(meta_test)
 part_size = int(max_line / n_parts)
 last_part = max_line % n_parts
start_end = [[x, x+part_size] for x in range(first_sig, max_line + first_sig, part_size)]
start_end = start_end[:-1] + [[start_end[-1][0], start_end[-1][0] + last_part]]
X_test = []
for start, end in start_end:
subset_test = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
for i in tqdm(subset_test.columns):
id_measurement, phase = meta_test.loc[int(i)]
subset_test_col = subset_test[i]
subset_trans = transform_ts(subset_test_col)
 X_test.append([i, id_measurement, phase, subset_trans] )
sub.to_csv('submission.csv', index=False )<load_from_csv>
submission = pd.read_csv('.. /input/sample_submission.csv' )
%%time
datadir = Path('/kaggle/input/google-quest-challenge')
 train = pd.read_csv(datadir/'train.csv')
 test = pd.read_csv(datadir/'test.csv')
 sample_submission = pd.read_csv(datadir/'sample_submission.csv' )<define_variables>
X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1)for i in range(0,len(X_test), 3)])
 X_test_input.shape
feature_columns = [col for col in train.columns if col not in sample_submission.columns]
print("Feature columns are " , feature_columns )<train_model>
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict(X_test_input, batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3 )
train0 = train.iloc[0]
print('URL : ', train0['url'])
 print('question_title: ', train0['question_title'])
 print('question_body : ', train0['question_body'] )<train_model>
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape
print('answer: ', train0['answer'] )<define_variables>
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head(10 )
<count_missing_values><EOS> 
submission.target.value_counts().sort_index()