Kaggle competition: VSB Power Line Fault Detection
<SOS> metric: MatthewsCorrelationCoefficient Kaggle data source: vsb-power-line-fault-detection<init_hyperparams>
RECALCULATE_PEAK_FEATURES = True
PREPROCESSED_DIR = '.. /input/vsb-1st-place-solution'
xgb_params = {'alpha': 13, 'colsample_bytree': 0.4,
'gamma': 0, 'lambda': 1,
'learning_rate': 0.1,
'max_depth': 4,
'n_estimators': 700,
'subsample': 0.8}
lgbm_params = {'subsample': 0.8, 'num_leaves': 50,
'n_estimators': 4000,
'min_gain_to_split': 10,
'max_depth': 5, 'max_bin': 40,
'learning_rate': 0.01,
'lambda_l1': 0.0 ,'lambda_l2': 0.6000000000000001,
'feature_fraction': 0.16000000000000003
}<split>
USE_SIMPLIFIED_VERSION = False
i = 0
train_pred_XGB = np.zeros(X.shape[0])
 train_pred_LGBM = np.zeros(X.shape[0])
 test_pred_XGB = []
test_pred_LGBM = []
fold = ['1st', '2nd', '3rd', '4th', '5th']
for train_index, test_index in k_fold.split(X):
print("TRAIN:", train_index[:10], "TEST:", test_index[:10])
 X_train, X_test = X.loc[train_index,:], X.loc[test_index,:]
y_train, y_test = y[train_index], y[test_index]
print("Train and make prediction for the {} Fold...".format(fold[i]))
xgb_model = xgb.XGBRegressor(random_state=79,booster='gbtree',
objective ="reg:squarederror",
**xgb_params)
 xgb_model.fit(X_train,y_train)
 lgbm_model = lgb.LGBMRegressor(random_state=79, objective = "regression",
metric = "rmse",**lgbm_params)
 lgbm_model.fit(X_train,y_train)
 train_pred_XGB[test_index] = xgb_model.predict(X_test)
 train_pred_LGBM[test_index] = lgbm_model.predict(X_test)
 test_pred_XGB.append(xgb_model.predict(test.drop(["ID","SalePrice"],
axis = 1)))
 test_pred_LGBM.append(lgbm_model.predict(test.drop(["ID","SalePrice"],
axis = 1)))
 i += 1<prepare_x_and_y>
if RECALCULATE_PEAK_FEATURES:
!conda update -y numba
Stacking_Train = pd.DataFrame({'XGBoost': train_pred_XGB, 'LightGBM':train_pred_LGBM,
'SalePrice': train.SalePrice})
 X = Stacking_Train.iloc[:,:2]
y = np.array(Stacking_Train['SalePrice'])
 Stacking_Test = pd.DataFrame({'XGBoost': sum(test_pred_XGB)/5,
'LightGBM':sum(test_pred_LGBM)/5,
} )<train_on_grid>
import copy
import gc
import os
import sys
import warnings
from IPython.core.display import display, HTML
import lightgbm as lgb
from matplotlib import pyplot as plt
import numba
import numpy as np
import pandas as pd
import pyarrow
import pyarrow.parquet as pq
import scipy.stats
import seaborn as sns
import sklearn
from sklearn.metrics import matthews_corrcoef, precision_recall_fscore_support
k_fold = RepeatedKFold(n_splits=5, n_repeats = 3, random_state=79)
 Final_est = ElasticNet(random_state = 79)
 Final_params = {'alpha': [100,1000],
'l1_ratio': np.arange(0, 1, 0.01)
 }
Final_model = GridSearchCV(Final_est, Final_params, scoring='neg_mean_absolute_error',
cv=k_fold, n_jobs=-1, verbose = 1)
 Final_model.fit(X,y)
 print(Final_model.best_params_)
 print("The CV Error is: " + str(-Final_model.best_score_))
best_model = Final_model.best_estimator_<train_model>
data_dir = '.. /input/vsb-power-line-fault-detection'
k_fold = KFold(n_splits=3, shuffle=True, random_state=79)
 i = 0
train_pred_best = np.zeros(X.shape[0])
 test_pred_best = []
rmseStack = np.zeros(3)
 fold = ['1st', '2nd', '3rd']
for train_index, test_index in k_fold.split(X):
print("TRAIN:", train_index[:10], "TEST:", test_index[:10])
 X_train, X_test = X.loc[train_index,:], X.loc[test_index,:]
y_train, y_test = y[train_index], y[test_index]
print("Train and make prediction for the {} Fold...".format(fold[i]))
best_model.fit(X_train,y_train)
 train_pred_best[test_index] = best_model.predict(X_test)
 rmseStack[i] = round(np.sqrt(mean_squared_error(y, best_model.predict(X))),2)
 test_pred_best.append(best_model.predict(Stacking_Test))
i += 1<compute_test_metric>
for p in [scipy, np, numba, sklearn, lgb, pyarrow, pd]:
print('{}=={}'.format(p.__name__, p.__version__))
print()
print(sys.version)
cv_score = np.mean(cross_val_score(best_model,X,y,scoring = 'neg_root_mean_squared_error', cv=k_fold,n_jobs = -1,verbose =0))
print('XGBoost RMSE for training set :', round(np.sqrt(mean_squared_error(y, train_pred_XGB)) ,2))
print('LGBM RMSE for training set :', round(np.sqrt(mean_squared_error(y, train_pred_LGBM)) ,2))
print('Model Stacking RMSE for training set :',round(np.sqrt(mean_squared_error(y, train_pred_best)) ,2))
print('Model Stacking Cross Validation RMSE for training set :', round(-cv_score,2))<prepare_x_and_y>
meta_train_df = pd.read_csv(data_dir + '/metadata_train.csv')
 meta_test_df = pd.read_csv(data_dir + '/metadata_test.csv' )
y_pred_final = sum(test_pred_best)/3 
<save_to_csv>
@numba.jit(nopython=True)
 def flatiron(x, alpha=100., beta=1):

 new_x = np.zeros_like(x)
 zero = x[0]
for i in range(1, len(x)) :
zero = zero*(alpha-beta)/alpha + beta*x[i]/alpha
new_x[i] = x[i] - zero
return new_x
submission = pd.DataFrame({"Id": test.ID, "SalePrice": y_pred_final})
 submission.to_csv("submission.csv", index = None)
 submission<import_modules>
@numba.jit(nopython=True)
 def drop_missing(intersect,sample):

 i=j=k=0
new_intersect=np.empty_like(intersect)
 while i< intersect.size and j < sample.size:
if intersect[i]==sample[j]:
new_intersect[k]=intersect[i]
k+=1
i+=1
j+=1
elif intersect[i]<sample[j]:
i+=1
else :
j+=1
return new_intersect[:k]
import lightgbm as lgb
import pandas as pd
import numpy as np
from datetime import datetime
from pathlib import Path
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import RepeatedStratifiedKFold<load_from_csv>
@numba.jit(nopython=True)
 def _local_maxima_1d_window_single_pass(x, w):
midpoints = np.empty(x.shape[0] // 2, dtype=np.intp)
 left_edges = np.empty(x.shape[0] // 2, dtype=np.intp)
 right_edges = np.empty(x.shape[0] // 2, dtype=np.intp)
 m = 0
i = 1
i_max = x.shape[0] - 1
while i < i_max:
if x[i - 1] < x[i]:
i_ahead = i + 1
while i_ahead < i_max and x[i_ahead] == x[i]:
i_ahead += 1
i_right = i_ahead - 1
f = False
i_window_end = i_right + w
while i_ahead < i_max and i_ahead < i_window_end:
if x[i_ahead] > x[i]:
f = True
break
i_ahead += 1
if x[i_ahead] < x[i]:
left_edges[m] = i
right_edges[m] = i_right
midpoints[m] =(left_edges[m] + right_edges[m])// 2
m += 1
i = i_ahead - 1
i += 1
midpoints = midpoints[:m]
left_edges = left_edges[:m]
right_edges = right_edges[:m]
return midpoints, left_edges, right_edges
@numba.jit(nopython=True)
 def local_maxima_1d_window(x, w=1):

 fm, fl, fr = _local_maxima_1d_window_single_pass(x, w)
 bm, bl, br = _local_maxima_1d_window_single_pass(x[::-1], w)
 bm = np.abs(bm - x.shape[0] + 1)[::-1]
bl = np.abs(bl - x.shape[0] + 1)[::-1]
br = np.abs(br - x.shape[0] + 1)[::-1]
m = drop_missing(fm, bm)
 return m
path=Path(".. /input/")
 train=pd.read_csv(path/"train.csv" ).drop("ID_code",axis=1)
 test=pd.read_csv(path/"test.csv" ).drop("ID_code",axis=1 )<categorify>
@numba.jit(nopython=True)
 def plateau_detection(grad, threshold, plateau_length=5):

 count = 0
loc = 0
for i in range(grad.shape[0]):
if grad[i] > threshold:
count += 1
if count == plateau_length:
loc = i - plateau_length
break
return loc
def augment(train,num_n=1,num_p=2):
newtrain=[train]
n=train[train.target==0]
for i in range(num_n):
newtrain.append(n.apply(lambda x:x.values.take(np.random.permutation(len(n)))))
 for i in range(num_p):
p=train[train.target>0]
newtrain.append(p.apply(lambda x:x.values.take(np.random.permutation(len(p)))))
 return pd.concat(newtrain)
<init_hyperparams>
def get_peaks(
 x,
window=25,
visualise=False,
visualise_color=None, 
):

 x_hp = flatiron(x, 100, 1)
 x_dn = np.abs(x_hp)
 peaks = local_maxima_1d_window(x_dn, window)
 heights = x_dn[peaks]
ii = np.argsort(heights)[::-1]
peaks = peaks[ii]
heights = heights[ii]
ky = heights
kx = np.arange(1, heights.shape[0]+1)
 conv_length = 9
grad = np.diff(ky, 1)/np.diff(kx, 1)
 grad = np.convolve(grad, np.ones(conv_length)/conv_length)
 grad = grad[conv_length-1:-conv_length+1]
knee_x = plateau_detection(grad, -0.01, plateau_length=1000)
 knee_x -= conv_length//2
if visualise:
plt.plot(grad, color=visualise_color)
 plt.axvline(knee_x, ls="--", color=visualise_color)
 peaks_x = peaks[:knee_x]
peaks_y = heights[:knee_x]
ii = np.argsort(peaks_x)
 peaks_x = peaks_x[ii]
peaks_y = peaks_y[ii]
return peaks_x, peaks_y, x_hp
param = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : -1,
"num_leaves" : 13,
"learning_rate" : 0.01,
"bagging_freq": 5,
"bagging_fraction" : 0.4,
"feature_fraction" : 0.05,
"min_data_in_leaf": 80,
"min_sum_heassian_in_leaf": 10,
"tree_learner": "serial",
"boost_from_average": "false",
"bagging_seed" : 10,
"verbosity" : 1,
}<train_model>
@numba.jit(nopython=True)
 def clip(v, l, u):

 if v < l:
v = l
elif v > u:
v = u
return v
result=np.zeros(test.shape[0])
 rskf = RepeatedStratifiedKFold(n_splits=5, n_repeats=5,random_state=10)
 for counter,(train_index, valid_index)in enumerate(rskf.split(train, train.target),1):
print(counter)
 t=train.iloc[train_index]
t=augment(t)
 trn_data = lgb.Dataset(t.drop("target",axis=1), label=t.target)
 v=train.iloc[valid_index]
val_data = lgb.Dataset(v.drop("target",axis=1), label=v.target)
 model = lgb.train(param, trn_data, 1000000, valid_sets = [val_data], verbose_eval=500, early_stopping_rounds = 3000)
 result +=model.predict(test)
<save_to_csv>
peak_features_names = [
'ratio_next',
'ratio_prev',
'small_dist_to_min',
'sawtooth_rmse',
]
num_peak_features = len(peak_features_names)
 @numba.jit(nopython=True)
 def create_sawtooth_template(sawtooth_length, pre_length, post_length):

 l = pre_length+post_length+1
st = np.zeros(l)
 for i in range(sawtooth_length+1):
j = pre_length+i
if j < l:
st[j] = 1 -(( 2./sawtooth_length)* i)
 return st
@numba.jit(nopython=True)
 def calculate_peak_features(px, x_hp0, ws=5, wl=25):

 features = np.ones(( px.shape[0], num_peak_features), dtype=np.float64)* np.nan
for i in range(px.shape[0]):
feature_number = 0
x = px[i]
x_next = x+1
x_prev = x-1
h0 = x_hp0[x]
ws_s = clip(x-ws, 0, 800000-1)
 ws_e = clip(x+ws, 0, 800000-1)
 wl_s = clip(x-wl, 0, 800000-1)
 wl_e = clip(x+wl, 0, 800000-1)
 ws_pre = x - ws_s
ws_post = ws_e - x
wl_pre = x - wl_s
wl_post = wl_e - x
if x_next < 800000:
h0_next = x_hp0[x_next]
features[i, feature_number] = np.abs(h0_next)/np.abs(h0)
 feature_number += 1
if x_prev >= 0:
h0_prev = x_hp0[x_prev]
features[i, feature_number] = np.abs(h0_prev)/np.abs(h0)
 feature_number += 1
x_hp_ws0 = x_hp0[ws_s:ws_e+1]
x_hp_wl0 = x_hp0[wl_s:wl_e+1]
x_hp_wl0_norm =(x_hp_wl0/np.abs(h0))
x_hp_ws0_norm =(x_hp_ws0/np.abs(h0))
x_hp_abs_wl0 = np.abs(x_hp_wl0)
 wl_max_0 = np.max(x_hp_abs_wl0)
 ws_opp_peak_i = np.argmin(x_hp_ws0*np.sign(h0))
features[i, feature_number] = ws_opp_peak_i - ws
feature_number += 1
x_hp_wl0_norm_sign = x_hp_wl0_norm * np.sign(h0)
 sawtooth_length = 3
st = create_sawtooth_template(sawtooth_length, wl_pre, wl_post)
 assert np.argmax(st)== np.argmax(x_hp_wl0_norm_sign)
 assert st.shape[0] == x_hp_wl0_norm_sign.shape[0]
features[i, feature_number] = np.mean(np.power(x_hp_wl0_norm_sign - st, 2))
feature_number += 1
if i == 0:
assert feature_number == num_peak_features
return features
submission = pd.read_csv(path/'sample_submission.csv')
 submission['target'] = result/counter
filename="{:%Y-%m-%d_%H_%M}_sub.csv".format(datetime.now())
 submission.to_csv(filename, index=False )<set_options>
def process_signal(
 data,
window=25, 
):

 px0, height0, x_hp0 = get_peaks(
 data.astype(np.float),
window=window, 
)
 f0 = calculate_peak_features(px0, x_hp0)
 return px0, height0, f0
sns.set_style('whitegrid')
<load_from_csv>
def process_measurement_peaks(data, signal_ids):

 res = []
sigid_res = []
assert data.shape[1] % 3 == 0
N = data.shape[1]//3
for i in range(N):
sigids = signal_ids[i*3:(i+1)*3]
x = data[:, i*3:(i+1)*3].astype(np.float)
 px0, height0, f0 = process_signal(
 x[:, 0] 
)
 px1, height1, f1 = process_signal(
 x[:, 1] 
)
 px2, height2, f2 = process_signal(
 x[:, 2] 
)
 if px0.shape[0] != 0:
res.append(np.hstack([
px0[:, np.newaxis],
height0[:, np.newaxis],
f0,
]))
sigid_res.append(np.ones(px0.shape[0], dtype=np.int)* sigids[0])
 if px1.shape[0] != 0:
res.append(np.hstack([
px1[:, np.newaxis],
height1[:, np.newaxis],
f1,
]))
sigid_res.append(np.ones(px1.shape[0], dtype=np.int)* sigids[1])
 if px2.shape[0] != 0:
res.append(np.hstack([
px2[:, np.newaxis],
height2[:, np.newaxis],
f2,
]))
sigid_res.append(np.ones(px2.shape[0], dtype=np.int)* sigids[2])
 return res, sigid_res
train_df = pd.read_csv('.. /input/train.csv')
 print('Rows: ',train_df.shape[0],'Columns: ',train_df.shape[1])
 train_df.info()<count_values>
def process_measurement(data_df, meta_df, fft_data):

 peaks, sigids = process_measurement_peaks(
 data_df.values,
meta_df['signal_id'].values, 
)
 peaks = np.concatenate(peaks)
 peaks = pd.DataFrame(
 peaks,
columns=['px', 'height'] + peak_features_names 
)
 peaks['signal_id'] = np.concatenate(sigids)
 phase_50hz = np.angle(fft_data, deg=False)
 phase_50hz = pd.DataFrame(
 phase_50hz,
columns=['phase_50hz'] 
)
 phase_50hz['signal_id'] = meta_df['signal_id'].values
peaks = pd.merge(peaks, phase_50hz, on='signal_id', how='left')
 dt =(20e-3/(800000))
f1 = 50
w1 = 2*np.pi*f1
peaks['phase_aligned_x'] =(np.degrees(
(w1*peaks['px'].values*dt)+ peaks['phase_50hz'].values 
)+ 90)% 360
peaks['Q'] = pd.cut(peaks['phase_aligned_x'], [0, 90, 180, 270, 360], labels=[0, 1, 2, 3])
 return peaks
train_df['target'].value_counts()<load_from_csv>
@numba.jit(nopython=True, parallel=True)
 def calculate_50hz_fourier_coefficient(data):

 n = 800000
assert data.shape[0] == n
omegas = np.exp(-2j * np.pi * np.arange(n)/ n ).reshape(n, 1)
 m_ = omegas ** np.arange(1, 2)
 m = m_.flatten()
res = np.zeros(data.shape[1], dtype=m.dtype)
 for i in numba.prange(data.shape[1]):
res[i] = m.dot(data[:, i].astype(m.dtype))
return res
test_df = pd.read_csv('.. /input/test.csv' )<drop_column>
%%time
if RECALCULATE_PEAK_FEATURES:
train_df = pq.read_pandas(
 data_dir + '/train.parquet'
).to_pandas()
train_fft = calculate_50hz_fourier_coefficient(train_df.values)
 train_peaks = process_measurement(
 train_df,
meta_train_df,
train_fft 
)
 del train_df, train_fft
gc.collect()
X_test1 = test_df.drop('ID_code',axis=1 )<drop_column>
if not RECALCULATE_PEAK_FEATURES:
train_peaks = pd.read_pickle(PREPROCESSED_DIR + '/train_peaks.pkl')
 train_peaks.to_pickle("train_peaks.pkl" )
X1 = train_df.drop(['ID_code','target'],axis=1 )<train_model>
train_peaks = pd.merge(train_peaks, meta_train_df[['signal_id', 'id_measurement', 'target']], on='signal_id', how='left' )
def _my_fit(y, y_pred, w):
diffs = np.abs(y - y_pred)
 return 100.* np.average(diffs, weights=w)
 my_fit = make_fitness(_my_fit, greater_is_better=False )<train_model>
%%time
if RECALCULATE_PEAK_FEATURES:
NUM_TEST_CHUNKS = 10
test_chunk_size = int(np.ceil(( meta_test_df.shape[0]/3.) /float(NUM_TEST_CHUNKS)) *3.)
test_peaks = []
for j in range(NUM_TEST_CHUNKS):
j_start = j*test_chunk_size
j_end =(j+1)*test_chunk_size
signal_ids = meta_test_df['signal_id'].values[j_start:j_end]
test_df = pq.read_pandas(
 data_dir + '/test.parquet',
columns=[str(c)for c in signal_ids]
).to_pandas()
test_fft = calculate_50hz_fourier_coefficient(test_df.values)
 p = process_measurement(
 test_df,
meta_test_df.iloc[j_start:j_end],
test_fft 
)
 test_peaks.append(p)
 print(j)
 del test_df
gc.collect()
del test_fft
gc.collect()
test_peaks = pd.concat(test_peaks)
function_set = ['add', 'sub', 'mul', 'div', 'log',
'sqrt', 'log', 'abs', 'neg', 'inv',
'max', 'min',
'sin', 'cos', 'tan' ]
gp = SymbolicRegressor(function_set=function_set, metric = my_fit,
verbose=1, generations = 3,
random_state=0, n_jobs=-1)
<predict_on_test>
if not RECALCULATE_PEAK_FEATURES:
test_peaks = pd.read_pickle(PREPROCESSED_DIR + '/test_peaks.pkl')
 test_peaks.to_pickle("test_peaks.pkl" )
X1a = np.array(X1)
 sam = X1a.shape[0]
col = X1a.shape[1]
X2a = np.zeros(( sam, col))
X_test1a = np.array(X_test1)
 sam_test = X_test1a.shape[0]
col_test = X_test1a.shape[1]
X_test2a = np.zeros(( sam_test, col_test))
for i in range(col):
X = np.delete(X1a,i,1)
 y = X1a[:,i]
gp.fit(X, y)
 X2a[:,i] = gp.predict(X)
 X = np.delete(X_test1a, i, 1)
 X_test2a[:,i] = gp.predict(X)
 X2 = pd.DataFrame(X2a)
 X_test2 = pd.DataFrame(X_test2a )<prepare_x_and_y>
test_peaks = pd.merge(test_peaks, meta_test_df[['signal_id', 'id_measurement']], on='signal_id', how='left' )
X = pd.concat([X1, X2], axis=1, sort=False)
 X_test = pd.concat([X_test1, X_test2], axis=1, sort=False)
 y = train_df['target']<choose_model_class>
def create_features(
 meta_df,
peaks_df, 
):
res = process(peaks_df, meta_df)
 return res
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42 )<init_hyperparams>
%%time
X_train_full = create_features(
 meta_train_df,
train_peaks,
)
params = {
'num_leaves': 13,
'min_data_in_leaf': 80,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.01,
'boosting': 'gbdt',
'bagging_freq': 5,
'bagging_fraction': 0.33,
'feature_fraction': 0.05,
'metric':'auc',
'verbosity': 1,
'min_sum_hessian_in_leaf': 10.0,
'num_threads': 12,
'tree_learner': 'serial',
'boost_from_average':'false'
} 
<train_model>
y_train =(
 meta_train_df.groupby('id_measurement')['target'].sum().round(0 ).astype(np.int)
 != 0
).astype(np.float)
 assert np.all(y_train.index.values == X_train_full.index.values )
prediction = np.zeros(len(X_test))
for fold_n,(train_index, valid_index)in enumerate(folds.split(X,y)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 model = lgb.train(params,train_data,num_boost_round=20000,
valid_sets = [train_data, valid_data],verbose_eval=300,early_stopping_rounds = 200)
 prediction += model.predict(X_test, num_iteration=model.best_iteration)/n_fold<train_model>
%%time
X_test_full = create_features(
 meta_test_df,
test_peaks,
)
train_pool = Pool(X,y)
 m = CatBoostClassifier(iterations=300,eval_metric="AUC", boosting_type = 'Ordered')
 m.fit(X,y,silent=True)
 y_pred1 = m.predict(X_test)
 m.best_score_<save_to_csv>
default_features_to_drop = [
'id_measurement',
'phase',
'fft_phase',
'signal_id',
]
sub = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub["target"] = prediction
sub.to_csv("submission.csv", index=False )<save_to_csv>
if USE_SIMPLIFIED_VERSION:
additional_features_to_drop = [
'abs_small_dist_to_min_mean_Q02',
'height_mean_Q02',
'ratio_next_mean_Q02',
'ratio_prev_mean_Q02',
]
else:
additional_features_to_drop = []
sub1 = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub1["target"] = y_pred1
sub1.to_csv("submission1.csv", index=False )<save_to_csv>
features_to_drop =(
 default_features_to_drop
+ additional_features_to_drop
)
sub2 = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub2["target"] =(prediction + y_pred1)/2
sub2.to_csv("submission2.csv", index=False )<set_options>
if USE_SIMPLIFIED_VERSION:
feature_names = sorted([c for c in X_train_full.columns if c not in features_to_drop])
 else:
feature_names = [
'peak_count_Q13',
'abs_small_dist_to_min_mean_Q02',
'height_mean_Q02',
'height_std_Q02',
'sawtooth_rmse_mean_Q02',
'peak_count_Q02',
'ratio_next_mean_Q02',
'ratio_prev_mean_Q02',
'peak_count_total'
]
%matplotlib inline
sns.set()
print(os.listdir(".. /input"))
warnings.filterwarnings("ignore" )<load_from_csv>
X_train = X_train_full[feature_names]
X_test = X_test_full[feature_names]
train=pd.read_csv('.. /input/train.csv')
 test=pd.read_csv('.. /input/test.csv')
 train.head()<count_missing_values>
@numba.jit
def mcc(tp, tn, fp, fn):
sup = tp * tn - fp * fn
inf =(tp + fp)*(tp + fn)*(tn + fp)*(tn + fn)
 if inf==0:
return 0
else:
return sup / np.sqrt(inf)
 def eval_mcc(y_true, y_prob, show=False):

 idx = np.argsort(y_prob)
 y_true_sort = y_true[idx]
n = y_true.shape[0]
nump = 1.0 * np.sum(y_true)
 numn = n - nump
tp = nump
tn = 0.0
fp = numn
fn = 0.0
best_mcc = 0.0
best_id = -1
prev_proba = -1
best_proba = -1
mccs = np.zeros(n)
 for i in range(n):
proba = y_prob[idx[i]]
if proba != prev_proba:
prev_proba = proba
new_mcc = mcc(tp, tn, fp, fn)
 if new_mcc >= best_mcc:
best_mcc = new_mcc
best_id = i
best_proba = proba
mccs[i] = new_mcc
if y_true_sort[i] == 1:
tp -= 1.0
fn += 1.0
else:
fp -= 1.0
tn += 1.0
if show:
y_pred =(y_prob >= best_proba ).astype(int)
 score = matthews_corrcoef(y_true, y_pred)
 plt.plot(mccs)
 return best_proba, best_mcc, y_pred
else:
return best_proba, best_mcc, None
print(train.isnull().values.any())
 print(test.isnull().values.any() )<create_dataframe>
def train(
 X_data,
y_data,
params,
feature_names,
verbose_eval=50,
kfold_random_state=23948,
predict=False,
calculate_mcc=False,
early_stopping_rounds=10,
num_folds=5,
num_iterations=10, 
):
params = copy.deepcopy(params)
 models = []
cv_scores = []
val_cv_scores = []
if calculate_mcc:
mcc_scores = []
val_mcc_scores = []
thresholds=[]
else:
mcc_scores = None
val_mcc_scores = None
thresholds = None
if predict:
yp_train = np.zeros(X_train.shape[0])
 yp_val = np.zeros(X_train.shape[0])
 yp_test = np.zeros(X_train.shape[0])
 else:
yp_train = None
yp_val = None
yp_test = None
for iter in range(num_iterations):
np.random.seed(kfold_random_state + iter)
 splits = np.zeros(X_data.shape[0], dtype=np.int)
 m = y_data == 1
splits[m] = np.random.randint(0, 5, size=m.sum())
 m = y_data == 0
splits[m] = np.random.randint(0, 5, size=m.sum())
 for fold in range(num_folds):
val_fold = fold
test_fold =(fold + 1)% num_folds
train_folds = [f for f in range(num_folds)if f not in [val_fold, test_fold]]
train_indices = np.where(np.isin(splits, train_folds)) [0]
val_indices = np.where(splits == val_fold)[0]
test_indices = np.where(splits == test_fold)[0]
trn = lgb.Dataset(
 X_data.values[train_indices],
y_data[train_indices],
feature_name=feature_names, 
)
 val = lgb.Dataset(
 X_data.values[val_indices],
y_data[val_indices],
feature_name=feature_names, 
)
 test = lgb.Dataset(
 X_data.values[test_indices],
y_data[test_indices],
feature_name=feature_names, 
)
 with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 model = lgb.train(
 params,
trn,
valid_sets=(trn, test, val),
valid_names=("train", "test", "validation"),
fobj=None,
feval=None,
early_stopping_rounds=early_stopping_rounds,
verbose_eval=verbose_eval, 
)
 yp = None
if predict:
yp = model.predict(X_train.values[train_indices])
 yp_train[train_indices] += yp
yp_val_fold = model.predict(X_train.values[val_indices])
 yp_val[val_indices] += yp_val_fold
yp_test_fold = model.predict(X_train.values[test_indices])
 yp_test[test_indices] += yp_test_fold
models.append(model)
 s = model.best_score['test']
cv_scores.append(s[list(s.keys())[0]])
 s = model.best_score['validation']
val_cv_scores.append(s[list(s.keys())[0]])
 if calculate_mcc:
best_proba, _, _ = eval_mcc(
 y_train.values[train_indices].astype(np.float),
yp,
False 
)
 best_mcc_val = matthews_corrcoef(
 y_train.values[val_indices].astype(np.float),
yp_val_fold.astype(np.float)> best_proba 
)
 best_mcc = matthews_corrcoef(
 y_train.values[test_indices].astype(np.float),
yp_test_fold.astype(np.float)> best_proba 
)
 val_mcc_scores.append(best_mcc_val)
 mcc_scores.append(best_mcc)
 thresholds.append(best_proba)
 if yp_train is not None:
yp_train /=(( num_folds - 2)* num_iterations)
 yp_val /= num_iterations
yp_test /= num_iterations
cv_scores = np.array(cv_scores)
 val_cv_scores = np.array(val_cv_scores)
 if calculate_mcc:
mcc_scores = np.array(mcc_scores)
 val_mcc_scores = np.array(val_mcc_scores)
 thresholds = np.array(thresholds)
 return(
 models,
cv_scores, val_cv_scores,
mcc_scores, val_mcc_scores,
yp_train, yp_val, yp_test, thresholds
)
es = ft.EntitySet(id='Santander')
 es.entity_from_dataframe(dataframe=train[features],
entity_id='train',
make_index = True,
index='index')
 fm, feat= ft.dfs(entityset=es,
target_entity='train',
trans_primitives=['multiply_numeric','add_numeric'],
max_depth=1 )<create_dataframe>
params = {
'objective': 'binary',
'boosting': 'gbdt',
'learning_rate': 0.01,
'num_leaves': 80,
'num_threads': 4,
'metric': 'binary_logloss',
'feature_fraction': 0.8,
'bagging_freq': 1,
'bagging_fraction': 0.8,
'seed': 23974,
'num_boost_round': 10000,
}
train=pd.concat(( train,fm.iloc[:,len(features):]),axis=1)
 del fm
gc.collect()
fm=pd.DataFrame()
train.info()<create_dataframe>
%%time 
(
 models,
cv_scores, val_cv_scores,
mcc_scores, val_mcc_scores,
yp_train, yp_val, yp_test, thresholds 
)= train(
 X_train,
y_train,
params,
feature_names,
verbose_eval=0,
kfold_random_state=123948,
predict=True,
calculate_mcc=True,
early_stopping_rounds=100,
num_iterations=25 
)
 print("")
 print("CV Logloss: {:.4f} +/- {:.4f}({:.4f})".format(cv_scores.mean() , cv_scores.std() /np.sqrt(cv_scores.shape[0]), cv_scores.std()))
print("CV Val Logloss: {:.4f} +/- {:.4f}({:.4f})".format(val_cv_scores.mean() , val_cv_scores.std() /np.sqrt(val_cv_scores.shape[0]), val_cv_scores.std()))
print("CV MCC: {:.4f} +/- {:.4f}({:.4f})".format(mcc_scores.mean() , mcc_scores.std() /np.sqrt(mcc_scores.shape[0]), mcc_scores.std()))
print("CV VAL MCC: {:.4f} +/- {:.4f}({:.4f})".format(val_mcc_scores.mean() , val_mcc_scores.std() /np.sqrt(val_mcc_scores.shape[0]), val_mcc_scores.std()))
print("Threshold {:.4f} +/- {:.4f}".format(thresholds.mean() , thresholds.std() /np.sqrt(thresholds.shape[0])))
 best_proba_meas, _, _ = eval_mcc(
 y_train.values.astype(np.float),
yp_train,
False 
)
 best_mcc_meas = matthews_corrcoef(
 y_train.values.astype(np.float),
yp_val > best_proba_meas 
)
 best_mcc_meas_test = matthews_corrcoef(
 y_train.values.astype(np.float),
yp_test > best_proba_meas 
)
 plt.show()
yp_train_df = pd.DataFrame(
 yp_train,
index=X_train_full.index 
)
 yp_val_df = pd.DataFrame(
 yp_val,
index=X_train_full.index 
)
 yp_test_df = pd.DataFrame(
 yp_test,
index=X_train_full.index 
)
 train_pred = meta_train_df[['id_measurement', 'signal_id', 'target']].copy()
train_pred = pd.merge(train_pred, yp_train_df, on='id_measurement')
 train_pred.rename({0:'prediction'}, axis=1, inplace=True)
 val_pred = meta_train_df[['id_measurement', 'signal_id', 'target']].copy()
val_pred = pd.merge(val_pred, yp_val_df, on='id_measurement')
 val_pred.rename({0:'prediction'}, axis=1, inplace=True)
 test_pred = meta_train_df[['id_measurement', 'signal_id', 'target']].copy()
test_pred = pd.merge(test_pred, yp_test_df, on='id_measurement')
 test_pred.rename({0:'prediction'}, axis=1, inplace=True)
 best_proba, _, _ = eval_mcc(
 train_pred['target'].values.astype(np.float),
train_pred['prediction'].values.astype(np.float),
False 
)
 best_mcc = matthews_corrcoef(
 val_pred['target'].values.astype(np.float),
val_pred['prediction'].values.astype(np.float)> best_proba 
)
 best_mcc_test = matthews_corrcoef(
 test_pred['target'].values.astype(np.float),
test_pred['prediction'].values.astype(np.float)> best_proba 
)
 plt.show()
print("MCC(measurement): {:.3f}".format(best_mcc_meas))
print("MCC Test(measurement): {:.3f}".format(best_mcc_meas_test))
print("Best Probability Threshold(measurement): {:.3f}".format(best_proba_meas))
print("MCC: {:.3f}".format(best_mcc))
print("MCC Test: {:.3f}".format(best_mcc_test))
print("Best Probability Threshold: {:.3f}".format(best_proba))
es.entity_from_dataframe(dataframe=test[features],
entity_id='test',
make_index = True,
index='index')
 fm_test, feat= ft.dfs(entityset=es,
target_entity='test',
trans_primitives=['multiply_numeric','add_numeric'],
max_depth=1 )<create_dataframe>
thresholds = np.linspace(0, 1, 100)[1:]
scores_train = []
scores_val = []
scores_test = []
for t in thresholds:
s_train = matthews_corrcoef(
 y_train.values.astype(np.float),
yp_train > t 
)
 s_val = matthews_corrcoef(
 y_train.values.astype(np.float),
yp_val > t 
)
 s_test = matthews_corrcoef(
 y_train.values.astype(np.float),
yp_test > t 
)
 scores_train.append(s_train)
 scores_val.append(s_val)
 scores_test.append(s_test)
 plt.plot(thresholds, scores_train)
 plt.plot(thresholds, scores_val)
 plt.plot(thresholds, scores_test)
 plt.axvline(best_proba)
 plt.axvline(thresholds[np.argmax(scores_val)], ls='--')
 plt.show()
print(np.max(scores_val), thresholds[np.argmax(scores_val)] )
test=pd.concat(( test,fm_test.iloc[:,len(features):]),axis=1)
 del fm_test
gc.collect()
fm_test=pd.DataFrame()
test.info()<define_variables>
train_pred['probability_thresholded'] =(train_pred['prediction'] > best_proba ).astype(np.int )
random_state = 222
kfold = StratifiedKFold(n_splits=12,shuffle=True,random_state=random_state)
 pred_val = np.zeros(len(train))
pred_test_baseline = np.zeros(len(test))
feature_base=train.columns.tolist() [2:202]<init_hyperparams>
train_pred.to_csv('train_results.csv', index=False )
param = {
'bagging_freq': 5,
'bagging_fraction': 0.4,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.01,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}<prepare_x_and_y>
importances = pd.DataFrame()
for fold_ in range(len(models)) :
model = models[fold_]
imp_df = pd.DataFrame()
imp_df['feature'] = X_train.columns
imp_df['gain'] = model.feature_importance('gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False )
X_train=train.iloc[:,2:]
y_train=train['target']
X_test=test.iloc[:,1:]<train_model>
prediction_errors = y_train.copy().to_frame()
prediction_errors.columns = ['target']
prediction_errors['error'] = np.abs(y_train.values - yp_train)
 prediction_errors = prediction_errors.sort_values('error' )
for foldIdx,(trn_idx, val_idx)in enumerate(kfold.split(X_train.loc[:,feature_base], y_train)) :
print("Fold {}".format(foldIdx))
lgbm_base=lgb.LGBMClassifier(n_estimators=100000,random_state=random_state,**param)
 lgbm_base.fit(X_train.iloc[trn_idx][feature_base],y_train[trn_idx],
eval_set=[(X_train.iloc[trn_idx][feature_base],y_train[trn_idx]),(X_train.iloc[val_idx][feature_base],y_train[val_idx])],
early_stopping_rounds = 3000,
verbose=5000)
 pred_val[val_idx] = lgbm_base.predict_proba(X_train.loc[val_idx,feature_base], num_iteration=lgbm_base.best_iteration_)[:,1]
pred_test_baseline += lgbm_base.predict_proba(X_test[feature_base],num_iteration=lgbm_base.best_iteration_)[:,1] / kfold.n_splits<compute_test_metric>
%%time
yp_test = np.zeros(X_test.shape[0])
 for j in range(len(models)) :
model = models[j]
yp_test += model.predict(X_test.values)/len(models )
print('AUC score: %.5f' % roc_auc_score(train['target'],pred_val))<define_variables>
test_submission = pd.DataFrame(
 yp_test,
index=X_test.index,
columns=['probability']
)
to_drop = [column for column in upper.columns if any(upper[column] > 0.7)]<define_variables>
test_submission['probability_thresholded'] =(yp_test > best_proba ).astype(np.int )
features_selected=[fea for fea in X_train.columns if fea not in to_drop]<train_model>
test_submission = pd.merge(
 meta_test_df[['id_measurement', 'signal_id']],
test_submission,
on='id_measurement',
how='left'
)
pred_val = np.zeros(len(train))
pred_test = np.zeros(len(test))
for foldIdx,(trn_idx, val_idx)in enumerate(kfold.split(X_train.loc[:,features_selected], y_train)) :
print("Fold {}".format(foldIdx))
lgbm=lgb.LGBMClassifier(n_estimators=100000,random_state=random_state,**param)
 lgbm.fit(X_train.iloc[trn_idx][features_selected],y_train[trn_idx],
eval_set=[(X_train.iloc[trn_idx][features_selected],y_train[trn_idx]),(X_train.iloc[val_idx][features_selected],y_train[val_idx])],
early_stopping_rounds = 3000,
verbose=5000)
 pred_val[val_idx] = lgbm.predict_proba(X_train.loc[val_idx,features_selected], num_iteration=lgbm.best_iteration_)[:,1]
pred_test += lgbm.predict_proba(X_test[features_selected],num_iteration=lgbm.best_iteration_)[:,1] / kfold.n_splits<compute_test_metric>
test_submission.to_csv('test_results.csv', index=False )
print('AUC score: %.5f' % roc_auc_score(y_train,pred_val))<save_to_csv>
submission = pd.read_csv(data_dir + '/sample_submission.csv' )
submission_baseline = pd.DataFrame({'ID_code': test.ID_code.values,
'target':pred_test_baseline})
 submission_baseline.to_csv("LGBM_baseline.csv", index=False)
 submission = pd.DataFrame({'ID_code': test.ID_code.values,
'target':pred_test})
 submission.to_csv("LGBM_V3.csv", index=False )<load_from_csv>
assert np.all(submission['signal_id'].values == test_submission['signal_id'].values)
 submission['target'] = test_submission['probability_thresholded'].values
EARLY_STOPPING=4000
NFOLDS=15
NSHUFFLES_1=2
NSHUFFLES_0=1
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 features = [c for c in train_df.columns if c not in ['ID_code', 'target']]
target = train_df['target']
num_folds = NFOLDS
folds = StratifiedKFold(n_splits=num_folds, shuffle=False, random_state=2319)
 oof_preds = np.zeros(( len(train_df), 1))
test_preds = np.zeros(( len(test_df), 1))
roc_cv =[]
param = {
'bagging_freq': 5,
'bagging_fraction': 0.335,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.0083,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}<categorify>
submission.to_csv('submission.csv', index=False )
def overshuff(df, y, n=2, m=1):

 df_1 = df[y==1]
y_1 = y[y==1]
randoms=np.random.randint(0, 999999, size=n)
 for i, rand in enumerate(randoms):
df_sh = df_1.apply(lambda x: x.sample(n=len(x), random_state=rand ).values)
 df = pd.concat([df, df_sh])
 y = pd.concat([y, y_1])
 print("Step {} of {}.The random state used is {}".format(i+1, n, rand))
print("The random state used for the final shuffling is {}".format(rand))
df_1 = df[y==0]
y_1 = y[y==0]
randoms=np.random.randint(0, 999999, size=m)
 for i, rand in enumerate(randoms):
df_sh = df_1.apply(lambda x: x.sample(n=len(x), random_state=rand ).values)
 df = pd.concat([df, df_sh])
 y = pd.concat([y, y_1])
 print("Step {} of {}.The random state used is {}".format(i+1, n, rand))
rand = np.random.randint(0, 999999, size=1)[0]
df, y = shuffle(df, y, random_state=rand)
 print("The random state used for the final shuffling is {}".format(rand))
return df, y<split>
submission.to_csv('submission.csv', index=False )
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values, target.values)) :
print("Fold idx:{}".format(fold_ + 1))
trn_x = train_df.iloc[trn_idx][features]
trn_y = target.iloc[trn_idx]
trn_x, trn_y = overshuff(trn_x, trn_y, NSHUFFLES_1, NSHUFFLES_0)
 print("Converting the data to lgbm format")
 trn_data = lgb.Dataset(trn_x, label=trn_y)
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx])
 print("Training the classifier")
 clf = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, val_data],
verbose_eval=5000, early_stopping_rounds = EARLY_STOPPING)
 print("Making predictions for the validation data")
 val_pred = clf.predict(train_df.iloc[val_idx][features],
num_iteration=clf.best_iteration)
 print("Computing the AUC score")
 roc_cv.append(roc_auc_score(target.iloc[val_idx], val_pred))
print("AUC = {}".format(roc_auc_score(target.iloc[val_idx], val_pred)))
 oof_preds[val_idx, :] = val_pred.reshape(( -1, 1))
print("Making predictions for the test data")
 test_fold_pred = clf.predict(test_df[features],
num_iteration=clf.best_iteration ).\
reshape(( -1, 1))
test_preds += test_fold_pred
test_preds /= num_folds
roc_score_1 = round(roc_auc_score(target.ravel() , oof_preds.ravel()), 5)
 roc_cv = np.array(roc_cv)
 roc_score = round(sum(roc_cv)/len(roc_cv), 5)
 st_dev = round(np.array(roc_cv ).std() , 5)
 print("Average of the folds' AUCs = {}".format(roc_score))
print("Combined folds' AUC = {}".format(roc_score_1))
print("The standard deviation = {}".format(st_dev))
print("Saving OOF predictions")
 oof_preds = pd.DataFrame(np.column_stack(( train_df.ID_code.values,
oof_preds.ravel())) ,
columns=['ID_code', 'target'])
 oof_preds.to_csv(( "LGBM_{}.csv" ).format(str(roc_score)) , index=False)
 print("Saving submission file")
 sub = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub["target"] = test_preds
sub.to_csv('submission_{}.csv'.format(str(roc_score)) , index=False )<set_options>
meta_train_df['target'].mean() ,(yp_train > best_proba ).astype(np.float ).mean() , submission['target'].mean()
<import_modules><EOS> 
for j in range(len(models)) :
models[j].save_model('model_{}.txt'.format(j))