Kaggle competition: VSB Power Line Fault Detection
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: vsb-power-line-fault-detection<load_from_csv>
seed = 1015
np.random.seed(seed)
 tf.set_random_seed(seed)
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<define_variables>
def denoise(signal, wavet = "db1", level=1):
coeff = pywt.wavedec(signal, wavet, mode="per")
 sigma = mad(coeff[-level])
 uthresh = sigma * np.sqrt(2*np.log(len(signal)))
 coeff[1:] =(pywt.threshold(i, value=uthresh, mode="hard")for i in coeff[1:])
 DY = pywt.waverec(coeff, wavet, mode="per")
 return DY
def sino(signal):
B, A = butter(1, 0.02, output='ba')
 sino_signal = filtfilt(B,A, signal)
 return signal-sino_signal
def wavelt(signal):
cA,cD = pywt.dwt(signal, "db1")
 return cA, cD
features = [c for c in train.columns if c not in ['ID_code', 'target']]
test_id_code = test['ID_code']
train_id_code = train['ID_code']
target = train['target']<define_variables>
N_SPLITS = 5
sample_size = 800000
df_train = train[features].copy()
df_test = test[features].copy()
train_target = target.values
ntrain = df_train.shape[0]
ntest = df_test.shape[0]<normalization>
def matthews_correlation(y_true, y_pred):

 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )
def hot_encoder(df, columns):
one_hot = {c: list(df[c].unique())for c in columns}
for c in one_hot:
for val in one_hot[c]:
df[c+'_oh_' + str(val)] =(df[c].values == val ).astype(np.int)
 return df
def scale_feat(df):
scaler = StandardScaler()
return pd.DataFrame(scaler.fit_transform(df), columns=df.columns)
 def rank_gauss(x):
N = x.shape[0]
temp = x.argsort()
rank_x = temp.argsort() / N
rank_x -= rank_x.mean()
rank_x *= 2
efi_x = erfinv(rank_x)
 efi_x -= efi_x.mean()
return efi_x
def df_inputSwapNoise(df, p):
n = df.shape[0]
idx = list(range(n))
swap_n = round(n*p)
 for col in df.columns:
arr = df[col].values
col_vals = np.random.permutation(arr)
 swap_idx = np.random.choice(idx, size= swap_n)
 arr[swap_idx] = np.random.choice(col_vals, size = swap_n)
 df[col] = arr
return df<randomize_order>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
print('Transforming data')
 feature_cols = [c for c in df_train.columns if c not in ['ID_code','target']]
keep_cols = [c for c in feature_cols]
df_all = pd.concat([df_train[keep_cols], df_test[keep_cols]])
 df_all_org = df_all.copy()
df_all_noise = df_inputSwapNoise(df_all, 0.15)
 data_all_org = df_all_org.values
data_all_noise = df_all_noise.values
cols = data_all_org.shape[1]
print('Final data with {} columns'.format(cols))<count_unique_values>
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()
for i in range(cols):
u = np.unique(data_all_org[:,i])
 if u.shape[0] > 3:
data_all_org[:,i] = rank_gauss(data_all_org[:,i])
 for i in range(cols):
u = np.unique(data_all_noise[:,i])
 if u.shape[0] > 3:
data_all_noise[:,i] = rank_gauss(data_all_noise[:,i])
 train_data_orig = data_all_org[0:ntrain,:]
test_data_orig = data_all_org[ntrain:,:]
train_data_noise = data_all_noise[0:ntrain,:]
test_data_noise = data_all_noise[ntrain:,:]
print(train_data_orig.shape)
 print(test_data_orig.shape)
 print(train_data_noise.shape)
 print(test_data_noise.shape )<concatenate>
max_num = 127
min_num = -128
print('Original data')
 all_data = np.vstack(( train_data_orig, test_data_orig))
print('Noise data')
 all_data_noise = np.vstack(( train_data_noise, test_data_noise))<choose_model_class>
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]
print('Creating neural net')
 model = Sequential()
model.add(Dense(units=1500, input_dim = all_data.shape[1], kernel_initializer=glorot_normal()))
model.add(Activation('relu'))
model.add(BatchNormalization())
 model.add(Dense(units=1500, kernel_initializer=glorot_normal()))
model.add(Activation('relu'))
model.add(BatchNormalization())
 model.add(Dense(units=1500, kernel_initializer=glorot_normal()))
model.add(Activation('relu'))
model.add(BatchNormalization())
 model.add(Dense(all_data.shape[1]))
model.add(Activation('linear'))
opt = keras.optimizers.Adam(lr=0.0009)
 model.compile(loss='mse', optimizer=opt )<train_model>
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts = denoise(ts)
 ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )
print('Training neural net')
 epochs = 15
chck = ModelCheckpoint('keras_dae.h5', monitor='loss', save_best_only=True)
 cb = [ EarlyStopping(monitor='loss', patience=100, verbose=2, min_delta=0), chck ]
model.fit(all_data_noise, all_data, batch_size=128, verbose=1, epochs=epochs, callbacks=cb)
 print('Applying neural net')
 train_data_transform = model.predict(train_data_orig)
 test_data_transform = model.predict(test_data_orig)
 print(train_data_transform.shape)
 print(test_data_transform.shape )<create_dataframe>
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y
train_data = pd.DataFrame(data=train_data_transform, columns=features)
 test_data = pd.DataFrame(data=test_data_transform, columns=features )<save_to_csv>
X = []
y = []
def load_all() :
total_size = len(df_train)
 for ini, end in [(0, int(total_size/2)) ,(int(total_size/2), total_size)]:
X_temp, y_temp = prep_data(ini, end)
 X.append(X_temp)
 y.append(y_temp)
 load_all()
X = np.concatenate(X)
 y = np.concatenate(y )
train_data.to_csv('train_dae.csv',index=False)
 test_data.to_csv('test_dae.csv',index=False )<init_hyperparams>
def model_lstm(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(input_shape[1] )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model
def model_(input_shape):
model = Sequential()
model.add(Conv2D(24, kernel_size=3,input_shape=(input_shape[1],input_shape[2],1)))
 model.add(ReLU())
 model.add(AveragePooling2D())
 model.add(Flatten())
 model.add(Dropout(0.7))
model.add(Dense(128))
model.add(BatchNormalization())
 model.add(ReLU())
 model.add(Dropout(0.7))
model.add(Dense(148))
model.add(ReLU())
 model.add(Dropout(0.7))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer="adam", metrics=[matthews_correlation])
 return model
def model_lstm2(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
x = Bidirectional(CuDNNGRU(128, return_sequences=True))(inp)
 x = Bidirectional(CuDNNGRU(64))(x)
 x = Attention(input_shape[1] )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model
param = {
'bagging_freq': 5,
'bagging_fraction': 0.335,
'boost_from_average':'true',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.0083,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}<train_model>
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=2019 ).split(X, y))
preds_val = []
y_val = []
histories = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = model_lstm(train_X.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 h = model.fit(train_X, train_y, batch_size=128, epochs=50, validation_data=[val_X, val_y], callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict(val_X, batch_size=512))
y_val.append(val_y)
 histories.append(h)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape
folds = StratifiedKFold(n_splits=15, shuffle=False, random_state=2319)
 oof = np.zeros(len(train_data))
predictions = np.zeros(len(test_data))
feature_importance_df = pd.DataFrame()
print(train_data.shape)
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_data.values, target.values)) :
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(train_data.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_data.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 1000000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 4000)
 oof[val_idx] = clf.predict(train_data.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_data[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<save_to_csv>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = K.eval(matthews_correlation(y_true.astype(np.float64),(y_proba > threshold ).astype(np.float64)))
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
return search_result
sub_df = pd.DataFrame({"ID_code":test_id_code})
 sub_df["target"] = predictions
sub_df.to_csv("submission1.csv", index=False)
<load_from_csv>
best_threshold = threshold_search(y_val, preds_val)['threshold']
s1 = pd.read_csv('.. /input/kernel-01000110-01010101-01001110/submission.csv' ).rename(columns={'target': 'target1'})
 s2 = pd.read_csv('.. /input/santander-ls/submission.csv' ).rename(columns={'target': 'target2'})
 s3 = pd.read_csv('.. /input/santander-improved-starter-solution/submission.csv' ).rename(columns={'target': 'target3'} )<save_to_csv>
%%time
meta_test = pd.read_csv('.. /input/metadata_test.csv' )
sub = pd.merge(s1, s2, how='left', on='ID_code')
 sub = pd.merge(sub, s3, how='left', on='ID_code')
 sub['target'] =(( sub['target1'] * 0.33)+(sub['target2'] * 0.33)+(sub['target3'] * 0.34)).clip(0,1)
 sub[['ID_code','target']].to_csv('submission.csv', index=False )<set_options>
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()
color = sns.color_palette()
%matplotlib inline
py.init_notebook_mode(connected=True)
 pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999<load_from_csv>
%%time
first_sig = meta_test.index[0]
n_parts = 10
max_line = len(meta_test)
 part_size = int(max_line / n_parts)
 last_part = max_line % n_parts
print(first_sig, n_parts, max_line, part_size, last_part, n_parts * part_size + last_part)
 start_end = [[x, x+part_size] for x in range(first_sig, max_line + first_sig, part_size)]
start_end = start_end[:-1] + [[start_end[-1][0], start_end[-1][0] + last_part]]
print(start_end)
 X_test = []
for start, end in start_end:
subset_test = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
for i in tqdm(subset_test.columns):
id_measurement, phase = meta_test.loc[int(i)]
subset_test_col = subset_test[i]
subset_trans = transform_ts(subset_test_col)
 X_test.append([i, id_measurement, phase, subset_trans] )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )<import_modules>
X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1)for i in range(0,len(X_test), 3)])
 np.save("X_test.npy",X_test_input)
 X_test_input.shape
from sklearn.model_selection import train_test_split
from sklearn.metrics import(roc_curve, auc, accuracy_score)
 from sklearn.model_selection import GridSearchCV<drop_column>
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()
test_ID = test_df['ID_code'].values
train_df = train_df.drop("ID_code",axis=1)
 test_df = test_df.drop("ID_code",axis=1 )<define_variables>
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict(X_test_input, batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3)
target_col = "target"<init_hyperparams>
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape
<define_search_space><EOS> 
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()