Kaggle competition: VSB Power Line Fault Detection
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: vsb-power-line-fault-detection<feature_engineering>
import pandas as pd
import pyarrow.parquet as pq
import os
import numpy as np
from keras.layers import *
from keras.models import Model
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from keras import backend as K
from keras import optimizers
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from keras.callbacks import *
for col in input_categories:
df_train[f'treated_{col}'] = df_train[col].progress_apply(lambda x: get_tree_tokens(x))
df_test[f'treated_{col}'] = df_test[col].progress_apply(lambda x: get_tree_tokens(x))
df[f'treated_{col}'] = df[col].progress_apply(lambda x: get_tree_tokens(x))<choose_model_class>
N_SPLITS = 5
sample_size = 800000
tokenizer = text.Tokenizer(lower=False )<prepare_x_and_y>
def matthews_correlation(y_true, y_pred):

 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )
X_train_question = df_train['question_body']
X_train_title = df_train['question_title']
X_train_answer = df_train['answer']
X_test_question = df_test['question_body']
X_test_title = df_test['question_title']
X_test_answer = df_test['answer']<train_model>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
tokenizer.fit_on_texts(list(X_train_title)+list(X_train_question)+list(X_train_answer)+list(X_test_title)+list(X_test_question)+list(X_test_answer))<choose_model_class>
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()
nlp = English()
sentencizer = nlp.create_pipe('sentencizer')
 nlp.add_pipe(sentencizer )<string_transform>
max_num = 127
min_num = -128
def split_document(texts):
all_sents = []
max_num_sentences = 0.0
for text in texts:
doc = nlp(text)
 sents=[]
for i,sent in enumerate(doc.sents):
sents.append(sent.text)
 all_sents.append(sents)
 return all_sents
X_train_question = split_document(X_train_question)
 X_train_answer = split_document(X_train_answer)
 X_test_question = split_document(X_test_question)
 X_test_answer = split_document(X_test_answer )<feature_engineering>
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]
def add_question_metadata_features(text):
doc=nlp(text)
 indirect = 0
choice_words=0
reason_explanation_words = 0
question_count = 0
for sent in doc.sents:
if '?' in sent.text and '?' == sent.text[-1]:
question_count += 1
for token in sent:
if token.text.lower() =='why':
reason_explanation_words+=1
elif token.text.lower() =='or':
choice_words+=1
if question_count==0:
indirect+=1
return np.array([indirect, question_count, reason_explanation_words, choice_words] )<count_duplicates>
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )
ans_user_and_category=df_train[df_train[['answer_user_name', 'category']].duplicated() ][['answer_user_name', 'category']].values
ans_user_and_category.shape<string_transform>
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y
def question_answer_author_same(df):
q_username = df['question_user_name']
a_username = df['answer_user_name']
author_same=[]
for i in range(len(df)) :
if q_username[i] == a_username[i]:
author_same.append(int(1))
else:
author_same.append(int(0))
return author_same 
<feature_engineering>
X = []
y = []
def load_all() :
total_size = len(df_train)
 for ini, end in [(0, int(total_size/2)) ,(int(total_size/2), total_size)]:
X_temp, y_temp = prep_data(ini, end)
 X.append(X_temp)
 y.append(y_temp)
 load_all()
X = np.concatenate(X)
 y = np.concatenate(y )
def add_external_features(df):
df['question_body'] = df['question_body'].progress_apply(lambda x: str(x))
df['question_body_num_words'] = df['question_body'].str.count('\S+')
 df['answer'] = df['answer'].progress_apply(lambda x: str(x))
df['answer_num_words'] = df['answer'].str.count('\S+')
 df['question_vs_answer_length'] = df['question_body_num_words']/df['answer_num_words']
df['q_a_author_same'] = question_answer_author_same(df)
 answer_user_cat = []
for i in tqdm(df[['answer_user_name', 'category']].values):
if i in ans_user_and_category:
answer_user_cat.append(int(1))
else:
answer_user_cat.append(int(0))
df['answer_user_cat'] = answer_user_cat
handmade_features=[]
for text in df['question_body'].values:
handmade_features.append(add_question_metadata_features(text))
return df, np.array(handmade_features)
<create_dataframe>
def model_lstm(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(input_shape[1] )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model
df_train, handmade_features = add_external_features(df_train)
 df_test, handmade_features_test = add_external_features(df_test)
 df_train = pd.concat([df_train,pd.DataFrame(handmade_features, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])],axis=1)
 df_test = pd.concat([df_test,pd.DataFrame(handmade_features_test, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])],axis=1 )<normalization>
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=2019 ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = model_lstm(train_X.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit(train_X, train_y, batch_size=128, epochs=50, validation_data=[val_X, val_y], callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict(val_X, batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape
num_words_scaler = MinMaxScaler()
df_train[['question_body_num_words', 'answer_num_words']] = num_words_scaler.fit_transform(df_train[['question_body_num_words', 'answer_num_words']].values)
 df_test[['question_body_num_words', 'answer_num_words']] = num_words_scaler.transform(df_test[['question_body_num_words', 'answer_num_words']].values )<categorify>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = K.eval(matthews_correlation(y_true.astype(np.float64),(y_proba > threshold ).astype(np.float64)))
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
return search_result
df=pd.concat([df,pd.get_dummies(df['host'], drop_first=False, prefix='host')],axis=1)
 df=pd.concat([df,pd.get_dummies(df['category'], drop_first=False, prefix='cat')],axis=1 )<define_variables>
best_threshold = threshold_search(y_val, preds_val)['threshold']
len(['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')] )<merge>
%%time
meta_test = pd.read_csv('.. /input/metadata_test.csv' )
df_train=pd.merge(df_train, df[['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]], how='inner', on='qa_id')
 df_test = pd.merge(df_test, df[['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]], how='inner', on='qa_id' )<categorify>
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()
<categorify>
%%time
first_sig = meta_test.index[0]
n_parts = 10
max_line = len(meta_test)
 part_size = int(max_line / n_parts)
 last_part = max_line % n_parts
print(first_sig, n_parts, max_line, part_size, last_part, n_parts * part_size + last_part)
 start_end = [[x, x+part_size] for x in range(first_sig, max_line + first_sig, part_size)]
start_end = start_end[:-1] + [[start_end[-1][0], start_end[-1][0] + last_part]]
print(start_end)
 X_test = []
for start, end in start_end:
subset_test = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
for i in tqdm(subset_test.columns):
id_measurement, phase = meta_test.loc[int(i)]
subset_test_col = subset_test[i]
subset_trans = transform_ts(subset_test_col)
 X_test.append([i, id_measurement, phase, subset_trans] )
<drop_column>
X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1)for i in range(0,len(X_test), 3)])
 np.save("X_test.npy",X_test_input)
 X_test_input.shape
df_train.drop(['host', 'category'], inplace=True, axis=1)
 df_test.drop(['host', 'category'], inplace=True, axis=1 )<string_transform>
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()
def _get_masks(tokens, max_seq_length):
if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+[0]*(max_seq_length-len(tokens))
def _get_segments(tokens, max_seq_length):
if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments=[]
first_sep=True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == '[SEP]':
current_segment_id=1
return segments+[0]*(max_seq_length-len(tokens))
def _trim_input_questions(title, question, max_sequence_length, t_max_len=30, q_max_len=479):
t = tokenizer2.tokenize(title)
 q = tokenizer2.tokenize(question)
 t_len = len(t)
 q_len = len(q)
 if(t_len+q_len+3)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if q_max_len > q_len:
q_new_len = q_len
else:
q_new_len = q_max_len
if t_new_len+q_new_len+3 > max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+q_new_len+3)))
 t = t[:t_new_len]
q = q[:q_new_len]
return t, q
def _get_ids(tokens, tokenizer, max_seq_length):
token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0]*(max_seq_length - len(token_ids))
return input_ids
def _convert_to_bert_inputs_questions(title, question,tokenizer, max_seq_length):
stoken = ['[CLS]']+title+['[SEP]']+question+['[SEP]']
input_ids = _get_ids(tokens=stoken, tokenizer=tokenizer,max_seq_length=max_seq_length)
 input_masks = _get_masks(tokens=stoken, max_seq_length=max_seq_length)
 input_segments = _get_segments(tokens=stoken, max_seq_length=max_seq_length)
 return [input_ids, input_masks, input_segments]
def compute_input_array_questions(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, col in tqdm(df[columns].iterrows()):
t, q = col['treated_question_title'], col['treated_question_body']
t,q = _trim_input_questions(t,q, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_questions(t,q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.array(input_ids, dtype=np.int32), np.array(input_masks, dtype=np.int32),
np.array(input_segments, dtype=np.int32)]
def _trim_input_answers(answer, max_sequence_length, a_max_len=510):
a = tokenizer2.tokenize(answer)
 a_len = len(a)
 if(a_len+2)> max_sequence_length:
if a_max_len > a_len:
a_new_len = a_len
else:
a_new_len = a_max_len
if a_new_len+2 > max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(a_new_len+2)))
 a = a[:a_new_len]
return a
def _convert_to_bert_inputs_answers(answer, tokenizer, max_seq_length):
stoken = ['[CLS]']+answer+['[SEP]']
input_ids = _get_ids(tokens=stoken, tokenizer=tokenizer,max_seq_length=max_seq_length)
 input_masks = _get_masks(tokens=stoken, max_seq_length=max_seq_length)
 input_segments = _get_segments(tokens=stoken, max_seq_length=max_seq_length)
 return [input_ids, input_masks, input_segments]
def compute_input_array_answers(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, col in tqdm(df[columns].iterrows()):
a = col['treated_answer']
a = _trim_input_answers(a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_answers(a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.array(input_ids, dtype=np.int32), np.array(input_masks, dtype=np.int32),
np.array(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns])
 def compute_spearmanr(trues, preds):
rhos=[]
for t, p in zip(trues.T, preds.T):
rhos.append(
 spearmanr(t,p ).correlation 
)
 return np.nanmean(rhos)
 class CustomCallback(keras.callbacks.Callback):
def __init__(self, valid_data, test_data, test_predictions=None, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.test_predictions = test_predictions
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions=[]
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(self.valid_outputs, np.average(self.valid_predictions, axis=0))
print(f"
validation rho: {round(rho_val,4)}")
 self.test_predictions.append(self.model.predict(self.test_inputs, batch_size=self.batch_size))<load_pretrained>
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict(X_test_input, batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3)
bert_config=BertConfig(unk_token="[QBODY]", pad_token="[ANS]" ).from_pretrained('.. /input/bert-tensorflow/bert-base-uncased-config.json',output_hidden_states=True)
 def bertModel() :
input_ids_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_word_ids_q')
 input_mask_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_masks_q')
 input_segments_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_segments_q')
 input_new_features = keras.layers.Input(( 9,), dtype=tf.float32, name='input_new_features')
 input_ids_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_word_ids_a')
 input_mask_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_masks_a')
 input_segments_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_segments_a')
 bert_model_q = TFBertModel.from_pretrained(pretrained_model_name_or_path='.. /input/bert-tensorflow/bert-base-uncased-tf_model.h5',config = bert_config)
 bert_model_a = TFBertModel.from_pretrained(pretrained_model_name_or_path='.. /input/bert-tensorflow/bert-base-uncased-tf_model.h5',config = bert_config)
 _, _, hidden_states_q = bert_model_q([input_ids_q,input_mask_q, input_segments_a])
 _, _, hidden_states_a = bert_model_a([input_ids_a,input_mask_a, input_segments_a])
 h12_q = tf.reshape(hidden_states_q[-1][:,0],(-1,1,768))
h11_q = tf.reshape(hidden_states_q[-2][:,0],(-1,1,768))
h10_q = tf.reshape(hidden_states_q[-3][:,0],(-1,1,768))
h09_q = tf.reshape(hidden_states_q[-4][:,0],(-1,1,768))
concat_hidden_q = keras.layers.Concatenate(axis=2 )([h12_q, h11_q, h10_q, h09_q])
 h12_a = tf.reshape(hidden_states_a[-1][:,0],(-1,1,768))
h11_a = tf.reshape(hidden_states_a[-2][:,0],(-1,1,768))
h10_a = tf.reshape(hidden_states_a[-3][:,0],(-1,1,768))
h09_a = tf.reshape(hidden_states_a[-4][:,0],(-1,1,768))
concat_hidden_a = keras.layers.Concatenate(axis=2 )([h12_a, h11_a, h10_a, h09_a])
 x = keras.layers.GlobalAveragePooling1D()(concat_hidden_q)
 x_a = keras.layers.GlobalAveragePooling1D()(concat_hidden_a)
 x = keras.layers.Concatenate()([x, input_new_features])
 x_a = keras.layers.Concatenate()([x_a,x, input_new_features])
 out = keras.layers.Dense(len([i for i in output_categories if i.startswith('question')]), activation='sigmoid', name='final_dense_output_q' )(x)
 out_a = keras.layers.Dense(len([i for i in output_categories if i.startswith('answer')]), activation='sigmoid', name='final_dense_output_a' )(x_a)
 out = keras.layers.Concatenate()([out,out_a])
 model = keras.models.Model(inputs=[input_ids_q, input_mask_q, input_segments_q,
input_new_features, input_ids_a, input_mask_a, input_segments_a]
, outputs=out)
 model.compile(loss='binary_crossentropy', optimizer = keras.optimizers.Adam(lr=3e-5))
return model<split>
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape
<randomize_order><EOS> 
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()