Kaggle competition: VSB Power Line Fault Detection
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: vsb-power-line-fault-detection<create_dataframe>
v_raw_train = pq.read_pandas('.. /input/train.parquet' ).to_pandas().values
meta_train = np.loadtxt('.. /input/metadata_train.csv', skiprows=1, delimiter=',')
 y_train = meta_train[:, 3].astype(bool)
 print(v_raw_train.shape )
submissions=pd.DataFrame({'ID_code':Index, 'target': sub_preds} )<save_to_csv>
def compute_spectra(v_raw, *, m = 1000):

 percentile =(100, 99, 95, 0, 1, 5)
 n = v_raw.shape[1]
length = v_raw.shape[0] // m
n_spectra = len(percentile)
 mean_signal = np.zeros(( n, length), dtype='float32')
 percentile_spectra = np.zeros(( n, length, n_spectra), dtype='float32')
 print('computing spectra...', flush=True)
 for i in tqdm(range(n)) :
v = v_raw[:, i].astype('float32' ).reshape(-1, m)/ 128.0
mean = np.mean(v, axis=1)
 s = np.abs(np.percentile(v, percentile, axis=1)- mean)
 h = np.percentile(s, 5.0)
 s = np.maximum(0.0, s - h)
 mean_signal[i, :] = mean
percentile_spectra[i, :, :] = s.T
d = {}
d['mean'] = mean_signal
d['percentile'] = percentile_spectra
return d
spec_train = compute_spectra(v_raw_train)
 print('done.' )
submissions.to_csv('submissions_lgb_samplingFEtuningbest1.csv', index=False )<split>
def max_windowed(spec, *, width=150, stride=10):

 n = spec.shape[0]
length = spec.shape[1]
nspec = spec.shape[2]
n_triplet = n // 3
max_spec3 = np.empty(( n_triplet, length, 3))
for i_triplet in range(n_triplet):
max_spec3[i_triplet, :, 0] = spec[3*i_triplet, :, 0]
max_spec3[i_triplet, :, 1] = spec[3*i_triplet + 1, :, 0]
max_spec3[i_triplet, :, 2] = spec[3*i_triplet + 2, :, 0]
x = tf.placeholder(tf.float32, [None, length, 3])
 K = np.ones(( width, 3, 1), dtype='float32')/ width
W_conv1 = tf.constant(K)
 h_conv1 = tf.nn.conv1d(x, W_conv1, stride=stride, padding='VALID')
 with tf.Session() as sess:
w = sess.run(h_conv1, feed_dict={x:max_spec3})
 imax = np.argmax(w[:, :, 0], axis=1)
 d = {}
d['w'] = w
d['ibegin'] = imax*stride
return d
peaks = max_windowed(spec_train['percentile'] )
val_preds = np.zeros(X.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 kf = StratifiedShuffleSplit(n_splits=10, test_size=0.2, random_state=42)
 for i,(train_index, test_index)in enumerate(kf.split(X, y)) :
X_train, y_train = X.iloc[train_index].copy() , y.iloc[train_index]
X_valid, y_valid = X.iloc[test_index].copy() , y.iloc[test_index].copy()
print("
Fold ", i)
 clf = LGBMClassifier(
 n_estimators=1704,
learning_rate=0.16624226726409647,
num_leaves=4,
colsample_bytree=.8,
subsample=.7,
subsample_freq= 7,
subsample_for_bin= 375140,
max_depth=5,
reg_alpha=1.081049236893711e-05,
reg_lambda=1.043686239159047,
min_split_gain=.01,
min_child_weight=4,
min_child_samples=22,
silent=-1,
verbose=-1, 
)
 clf.fit(X_train, y_train,
eval_set= [(X_train, y_train),(X_valid, y_valid)],
eval_metric='auc', verbose=100, early_stopping_rounds=100 
)
 val_preds[test_index] = clf.predict_proba(X_valid, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(X_test.loc[:,'var_0':], num_iteration=clf.best_iteration_)[:, 1] / kf.n_splits<predict_on_test>
def compute_features(v_raw, spec=None):

 if spec is None:
spec = compute_spectra(v_raw)
 v_spec = spec['percentile']
shape = v_spec.shape
n = shape[0]
length = shape[1]
nspec = shape[2]
n_triplet = n // 3
spec3 = np.empty(( n_triplet, length, nspec, 3))
for i_triplet in range(n_triplet):
spec3[i_triplet, :, :, 0] = v_spec[3*i_triplet, :, :]
spec3[i_triplet, :, :, 1] = v_spec[3*i_triplet + 1, :, :]
spec3[i_triplet, :, :, 2] = v_spec[3*i_triplet + 2, :, :]
width = 150
peaks = max_windowed(v_spec, width=width)
 n_feature4 = 3
X = np.empty(( n_triplet, n_feature4*nspec*3 + 3))
X4 = np.empty(( n_triplet, n_feature4, nspec, 3))
for i_triplet in range(n_triplet):
X4[i_triplet, 0, :, :] = np.max(spec3[i_triplet, :, :, :], axis=0)
 ibegin = peaks['ibegin'][i_triplet]
iend = ibegin + width
imid = ibegin + width // 2
X4[i_triplet, 1, :, :] = np.mean(spec3[i_triplet, ibegin:iend, :, :], axis=0)
 X4[i_triplet, 2, :, :] = np.max(spec3[i_triplet, ibegin:iend, :, :], axis=0)
 X[i_triplet, 0] = spec['mean'][3*i_triplet, imid]
X[i_triplet, 1] = spec['mean'][3*i_triplet + 1, imid]
X[i_triplet, 2] = spec['mean'][3*i_triplet + 2, imid]
shape = X4.shape
X[:, 3:] = X4.reshape(shape[0], shape[1]*shape[2]*shape[3])
 return X
X_all3 = compute_features(v_raw_train, spec_train)
 y_all3 = np.sum(y_train.reshape(-1, 3), axis=1)>= 2
print('Three phases are combined into one training data; the shapes are, therefore,')
 print(X_all3.shape, y_all3.shape )
val_preds[test_index] = clf.predict_proba(X_valid, num_iteration=clf.best_iteration_)[:,1]<define_variables>
n_splits = 5
splits = list(sklearn.model_selection.StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=2019 ).split(X_all3, y_all3))
models = []
scores = np.zeros(n_splits)
 print('Training...')
 print('MCC training & cv')
 for i,(idx_train, idx_cv)in enumerate(splits):
X_train = X_all3[idx_train, :]
y_train = y_all3[idx_train]
X_cv = X_all3[idx_cv, :]
y_cv = y_all3[idx_cv]
learning_rate = 0.006
model = CatBoostClassifier(learning_rate=learning_rate, od_type='IncToDec',
loss_function='Logloss', use_best_model=True, eval_metric='MCC')
 model.fit(X_train, y_train.astype(float), silent=True,
eval_set=(X_cv, y_cv.astype(float)))
 y_predict_train = model.predict(X_train)
 y_predict_cv = model.predict(X_cv)
 score_train = sklearn.metrics.matthews_corrcoef(y_train, y_predict_train)
 score_cv = sklearn.metrics.matthews_corrcoef(y_cv, y_predict_cv)
 models.append(model)
 scores[i] = score_cv
print('%d %.3f %.3f' %(i, score_train, score_cv))
print('CV scores %.3f ± %.3f' %(np.mean(scores), np.std(scores)) )
predictions_lgbm_01 = np.where(val_preds[test_index] > 0.5, 1, 0 )<compute_train_metric>
if 'v_raw_train' in globals() :
del v_raw_train
id_test = np.loadtxt('.. /input/metadata_test.csv', skiprows=1, delimiter=',')[:, 0].astype(int)
 n_test = len(id_test)
 X_tests = []
n_subset = 4
nread = 0
for i_subset in range(n_subset):
ibegin = 8712 + 3*int(n_test // 3 *(i_subset/n_subset))
iend = 8712 + 3*int(n_test // 3 *(( i_subset + 1)/n_subset))
print('Loading %d/%d; signal_id %d - %d...' %(i_subset, n_subset, ibegin, iend))
v_raw_test = pq.read_pandas('.. /input/test.parquet',
columns=[str(i)for i in range(ibegin, iend)] ).to_pandas().values
nread += v_raw_test.shape[1]
X = compute_features(v_raw_test)
 X_tests.append(X)
 print('%d/%d test data processed.' %(nread, n_test))
del v_raw_test
X_test = np.concatenate(X_tests, axis=0)
 assert(X_test.shape[0] == id_test.shape[0] // 3)
 del X_tests
print('X_test computation done.shape', X_test.shape )
lgb_F1 = f1_score(y_valid, predictions_lgbm_01, average = 'weighted')
 print("The Light GBM F1 for Validation Data is", lgb_F1 )<compute_test_metric>
y_test_probas = np.empty(( X_test.shape[0], n_splits))
assert(len(models)== n_splits)
 for i, model in enumerate(models):
y_test_probas[:, i] = model.predict_proba(X_test)[:, 1]
y_test_proba = np.mean(y_test_probas, axis=1)
 y_submit = np.repeat(y_test_proba > 0.25, 3)
 print('Positive fraction %d/%d = %.3f' %(
 np.sum(y_submit), len(y_submit), np.sum(y_submit)/len(y_submit)) )
<find_best_score><EOS> 
assert(len(id_test)== len(y_submit))
n = len(id_test)
 with open('submission.csv', 'w')as f:
f.write('signal_id,target
')
 for i in range(n):
f.write('%d,%d
' %(id_test[i], int(y_submit[i])))
 print('submission.csv written' )