Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<init_hyperparams>
cf.go_offline()
py.init_notebook_mode()
cf.getThemes()
cf.set_config_file(theme='ggplot')
 warnings.simplefilter('ignore')
 pd.plotting.register_matplotlib_converters()
sns.mpl.rc('figure',figsize=(16, 6))
plt.style.use('ggplot')
 sns.set_style('darkgrid')
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=12,
zoom_range = 0.25,
shear_range=15,
width_shift_range=0.25,
height_shift_range=0.25,
horizontal_flip=False,
vertical_flip=False,
data_format="channels_last"
)<find_best_params>
base = os.path.abspath('/kaggle/input/liverpool-ion-switching/')
 train = pd.read_csv(os.path.join(base + '/train.csv'))
test = pd.read_csv(os.path.join(base + '/test.csv'))
def epoch_cv(df_cv_per_epoch_val_acc, fold_num):
if fold_num==0:
df_cv_per_epoch_val_acc.index += 1
df_cv_per_epoch_val_acc['mean_val_acc'] = df_cv_per_epoch_val_acc.mean(axis=1)
 num_epochs = df_cv_per_epoch_val_acc.shape[0]
for i in range(1, num_epochs+1):
if i<moving_average_period+1:
df_cv_per_epoch_val_acc.at[i, 'moving_average'] = df_cv_per_epoch_val_acc.iloc[:i]['mean_val_acc'].mean()
else:
df_cv_per_epoch_val_acc.at[i, 'moving_average'] = df_cv_per_epoch_val_acc.iloc[i-moving_average_period:i]['mean_val_acc'].mean()
best_epoch_cv = df_cv_per_epoch_val_acc['mean_val_acc'].idxmax()
best_epoch_cv_by_moving_average = df_cv_per_epoch_val_acc['moving_average'].idxmax()
return df_cv_per_epoch_val_acc, best_epoch_cv, best_epoch_cv_by_moving_average<define_variables>
def add_bathing_to_data(df : pd.DataFrame)-> pd.DataFrame :
batches = df.shape[0] // 500000
df['batch'] = 0
for i in range(batches):
idx = np.arange(i*500000,(i+1)*500000)
 df.loc[idx, 'batch'] = i + 1
return df
def p5(x : pd.Series)-> pd.Series : return x.quantile(0.05)
 def p95(x : pd.Series)-> pd.Series : return x.quantile(0.95)
Run_CV = "Y"
Run_Holdout_Validation = "Y"
Run_Kaggle_Pred = "Y"
n_epochs = 75
my_batch_size = 128
my_verbose = 0
K = 4
len_test = len(X_test)
 moving_average_period = 10
mean_chart_lower_epoch_bound = 30<choose_model_class>
train = add_bathing_to_data(train)
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', patience=4, verbose=my_verbose, factor=0.55, min_lr=0.00001 )<train_model>
train.groupby('batch')[['signal','open_channels']].agg(['min', 'max', 'median', p5, p95])
print(" Epochs: " + str(n_epochs))
print(" Cross Validation Requested: " + Run_CV)
 print(" Kaggle Prediction Requested: " + Run_Kaggle_Pred )<train_on_grid>
train.groupby('open_channels')[['signal','batch']].agg(['min', 'max', 'median', p5, p95])
if Run_CV=="Y":
print(" Batch Size: " + str(my_batch_size))
print(" Number of K-Folds: " + str(K))
print(( 'Fold Preparation: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 kfold = StratifiedKFold(n_splits = K,
random_state = 2007,
shuffle = True)
 oof_pred = None
df_cv_per_epoch_train_acc = pd.DataFrame()
df_cv_per_epoch_val_acc = pd.DataFrame()
print(( 'KFold Model Starting: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 for i,(f_ind, outf_ind)in enumerate(kfold.split(X_train, y_train)) :
my_optimiser = Adam(lr=0.004, beta_1=0.9, beta_2=0.999, epsilon=my_epsilon, decay=0.0, amsgrad=False)
 X_train_f, X_val_f = X_train.loc[f_ind].copy() , X_train.loc[outf_ind].copy()
X_train_f = X_train_f.astype('float32')/ 255.
 X_train_f = X_train_f.values.reshape(X_train_f.shape[0], 28, 28, 1 ).astype('float32')
 X_val_f = X_val_f.astype('float32')/ 255.
 X_val_f = X_val_f.values.reshape(X_val_f.shape[0], 28, 28, 1 ).astype('float32')
 input_shape = X_train_f.shape[1:]
nnet_model = build_network(input_shape)
 y_train_f, y_val_f = y_train[f_ind], y_train[outf_ind]
y_train_f = y_train_f.values
y_val_f = y_val_f.values
y_val_f_series = y_val_f
y_train_f = to_categorical(y_train_f, num_classes = nb_classes)
 y_val_f = to_categorical(y_val_f, num_classes = nb_classes)
 print(( 'Augmenting Data: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 datagen.fit(X_train_f)
 print(( 'Model Fitting: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 print('Fold: ' + str(i))
history = nnet_model.fit_generator(datagen.flow(X_train_f,y_train_f, batch_size=my_batch_size), epochs=n_epochs, verbose=my_verbose,
steps_per_epoch=X_train.shape[0] // my_batch_size, validation_data=(X_val_f,y_val_f), callbacks=[learning_rate_reduction])
 df_cv_per_epoch_train_acc['fold_'+str(i)] = history.history['accuracy']
df_cv_per_epoch_val_acc['fold_'+str(i)] = history.history['val_accuracy']
y_preds = nnet_model.predict(X_val_f)
 if Run_Holdout_Validation=="Y":
if i==0:
holdout_preds = nnet_model.predict(X_holdout)
 else:
holdout_preds = holdout_preds + nnet_model.predict(X_holdout)
 if Run_Kaggle_Pred=="Y":
if i==0:
test_preds = nnet_model.predict(X_test)
 else:
test_preds = test_preds + nnet_model.predict(X_test)
 y_preds_series = np.argmax(y_preds,axis = 1)
 y_preds_series = pd.Series(y_preds_series,name="label")
 fold_accuracy = accuracy_score(y_val_f_series, y_preds_series)
 print(" Fold Accuracy = %3.6f"%(fold_accuracy))
if oof_pred is None:
oof_pred = y_preds_series
oof_pred_ids = outf_ind
else:
oof_pred = np.hstack(( oof_pred, y_preds_series))
oof_pred_ids = np.hstack(( oof_pred_ids, outf_ind))
df_cv_per_epoch_train_acc, best_train_epoch_cv, best_train_epoch_cv_by_moving_average = epoch_cv(df_cv_per_epoch_train_acc, i)
 df_cv_per_epoch_val_acc, best_epoch_cv, best_epoch_cv_by_moving_average = epoch_cv(df_cv_per_epoch_val_acc, i)
 print("Best Single Epoch: " + str(best_epoch_cv))
df_cv_per_epoch_val_acc['mean_train_acc'] = df_cv_per_epoch_train_acc['mean_val_acc']
PlotAcc(history, n_epochs)
 df_cv_per_epoch_val_acc.to_csv('df_cv_epoch_{:%Y%m%d%H%M%S}.csv'.format(datetime.datetime.now()), index=True)
 oof_pred = np.column_stack(( oof_pred_ids, oof_pred))
df_oof_pred = pd.DataFrame(oof_pred,index=oof_pred[:,0])
 df_oof_pred.columns = ['id', 'label']
df_oof_pred = df_oof_pred.sort_values(by=('id'), ascending=True)
 y_valid_pred = df_oof_pred['label'].values
oof_accuracy = accuracy_score(y_train, y_valid_pred)
 print(" Overall Out-of-Fold Accuracy = %3.4f"%(oof_accuracy))
else:
print("Cross-Validation skipped")
 if Run_Kaggle_Pred=="Y":
test_preds = test_preds / K
results = np.argmax(test_preds,axis = 1)
 results = pd.Series(results,name="label")
 print(( 'Writing Prediction: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())))
 submission = pd.concat([pd.Series(range(0,len_test),name = "id"),results],axis = 1)
 submission.to_csv("submission.csv",index=False )<define_variables>
train.groupby(['batch','open_channels'])[['signal']].agg(['min', 'max', 'median', p5, p95])
lbound = min(n_epochs-1, mean_chart_lower_epoch_bound)
 ubound = max(n_epochs, lbound)+ 1<compute_test_metric>
def maddest(d, axis=None):
return np.mean(np.absolute(d - np.mean(d, axis)) , axis)
 def high_pass_filter(x, low_cutoff=1000, sample_rate=10000):
nyquist = 0.5 * sample_rate
norm_low_cutoff = low_cutoff / nyquist
print(norm_low_cutoff)
 sos = butter(10, Wn=[norm_low_cutoff], btype='highpass', output='sos')
 filtered_sig = signal.sosfilt(sos, x)
 return filtered_sig
def denoise_signal(x, wavelet='db4', level=1):
coeff = pywt.wavedec(x, wavelet, mode="per")
 sigma =(1/0.6745)* maddest(coeff[-level])
 uthresh = sigma * np.sqrt(2*np.log(len(x)))
 coeff[1:] =(pywt.threshold(i, value=uthresh, mode='hard')for i in coeff[1:])
 return pywt.waverec(coeff, wavelet, mode='per' )
if Run_Holdout_Validation=="Y":
holdout_preds = holdout_preds / K
results_holdout = np.argmax(holdout_preds,axis = 1)
 results_holdout = pd.Series(results_holdout,name="label")
 holdout_accuracy = accuracy_score(y_holdout, results_holdout)
 print(" Holdout Accuracy = %3.5f"%(holdout_accuracy))<train_model>
train = add_rooling_data(train )
cv_weight = 7
holdout_weight = 1
blended_validation_accuracy =(( cv_weight * oof_accuracy)+(holdout_weight * holdout_accuracy)) /(cv_weight + holdout_weight)
 print("Blended Validation Accuracy = %3.5f"%(blended_validation_accuracy))<train_model>
train = pd.read_csv(os.path.join(base + '/train.csv'))
test = pd.read_csv(os.path.join(base + '/test.csv'))
def features(df):
df = df.sort_values(by=['time'] ).reset_index(drop=True)
 df.index =(( df.time * 10_000)- 1 ).values
df['batch'] = df.index // 25_000
df['batch_index'] = df.index -(df.batch * 25_000)
 df['batch_slices'] = df['batch_index'] // 2500
df['batch_slices2'] = df.apply(lambda r: '_'.join([str(r['batch'] ).zfill(3), str(r['batch_slices'] ).zfill(3)]), axis=1)
 for c in ['batch','batch_slices2']:
d = {}
d['mean'+c] = df.groupby([c])['signal'].mean()
d['median'+c] = df.groupby([c])['signal'].median()
d['max'+c] = df.groupby([c])['signal'].max()
d['min'+c] = df.groupby([c])['signal'].min()
d['std'+c] = df.groupby([c])['signal'].std()
d['mean_abs_chg'+c] = df.groupby([c])['signal'].apply(lambda x: np.mean(np.abs(np.diff(x))))
d['abs_max'+c] = df.groupby([c])['signal'].apply(lambda x: np.max(np.abs(x)))
 d['abs_min'+c] = df.groupby([c])['signal'].apply(lambda x: np.min(np.abs(x)))
 d['range'+c] = d['max'+c] - d['min'+c]
d['maxtomin'+c] = d['max'+c] / d['min'+c]
d['abs_avg'+c] =(d['abs_min'+c] + d['abs_max'+c])/ 2
for v in d:
df[v] = df[c].map(d[v].to_dict())
 df['signal_shift_+1'] = [0,] + list(df['signal'].values[:-1])
 df['signal_shift_-1'] = list(df['signal'].values[1:])+ [0]
for i in df[df['batch_index']==0].index:
df['signal_shift_+1'][i] = np.nan
for i in df[df['batch_index']==49999].index:
df['signal_shift_-1'][i] = np.nan
for c in [c1 for c1 in df.columns if c1 not in ['time', 'signal', 'open_channels', 'batch', 'batch_index', 'batch_slices', 'batch_slices2']]:
df[c+'_msignal'] = df[c] - df['signal']
return df
train = features(train)
 test = features(test)
 col = [c for c in train.columns if c not in ['time', 'open_channels', 'batch', 'batch_index', 'batch_slices', 'batch_slices2']]
x1, x2, y1, y2 = model_selection.train_test_split(train[col], train['open_channels'], test_size=0.3, random_state=7)
 target = train['open_channels']
train = train[col]
def MacroF1Metric(preds, dtrain):
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = metrics.f1_score(labels, preds, average = 'macro')
 return('MacroF1Metric', score, True)
print(( 'Run Completed: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())) )<set_options>
params = {'learning_rate': 0.1, 'max_depth': -1, 'num_leaves':2**7+1, 'metric': 'rmse', 'random_state': 7, 'n_jobs':-1, 'sample_fraction':0.33}
model = lgb.train(params, lgb.Dataset(x1, y1), 22222, lgb.Dataset(x2, y2), verbose_eval=0, early_stopping_rounds=250, feval=MacroF1Metric)
 preds_lgb =(model.predict(test[col], num_iteration=model.best_iteration)).astype(np.float16)
 oof_lgb =(model.predict(train, num_iteration=model.best_iteration)).astype(np.float16)
 gc.collect()
warnings.filterwarnings('ignore' )<load_from_csv>
params = {'colsample_bytree': 0.375,'learning_rate': 0.1,'max_depth': 10, 'subsample': 1, 'objective':'reg:squarederror',
'eval_metric':'rmse', 'n_estimators':22222, 'tree_method':'gpu_hist',}
train_set = xgb.DMatrix(x1, y1)
 val_set = xgb.DMatrix(x2, y2)
 model = xgb.train(params, train_set, num_boost_round=2222, evals=[(train_set, 'train'),(val_set, 'val')],
verbose_eval=0, early_stopping_rounds=250)
 preds_xgb = model.predict(xgb.DMatrix(test[col]), ntree_limit=model.best_ntree_limit)
 oof_xgb =(model.predict(xgb.DMatrix(train), ntree_limit=model.best_ntree_limit)).astype(np.float16)
 del train_set, val_set; gc.collect()
train = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 test = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')
 sub = pd.read_csv('/kaggle/input/Kannada-MNIST/sample_submission.csv')
 print('Our train set have {} rows and {} columns'.format(train.shape[0], train.shape[1]))
print('Our test set have {} rows and {} columns'.format(test.shape[0], test.shape[1]))<count_missing_values>
model = CatBoostRegressor(task_type = 'GPU', iterations=22222, learning_rate=0.1, random_seed = 7, depth=7, eval_metric='RMSE')
 train_dataset = Pool(x1, y1)
 eval_dataset = Pool(x2, y2)
 model.fit(train_dataset, eval_set=eval_dataset, verbose=0, early_stopping_rounds=250)
 preds_cb =(model.predict(test[col])).astype(np.float16)
 oof_cb =(model.predict(train)).astype(np.float16)
 del train_dataset, eval_dataset, model; gc.collect()
train.isnull().sum().sum()<split>
final_preds = 0.4 * preds_lgb + 0.4 * preds_xgb + 0.2 * preds_cb
final_oof = 0.4 * oof_lgb + 0.4 * oof_xgb + 0.2 * oof_cb
def preprocessing(train, test):
IMG_SIZE = 28
img_train = train.drop(['label'], axis = 1 ).values.reshape(-1, IMG_SIZE, IMG_SIZE, 1 ).astype('float32')
 img_test = test.drop(['id'], axis = 1 ).values.reshape(-1, IMG_SIZE, IMG_SIZE, 1 ).astype('float32')
 img_y = train['label'].values
img_train /= 255
img_test /= 255
x_train, x_val, y_train, y_val = train_test_split(img_train, img_y, test_size = 0.20)
 print('Our transformed train set have the following dimension: ', x_train.shape)
 print('Our transformed valid set have the following dimension: ', x_val.shape)
 return img_test, x_train, x_val, y_train, y_val<choose_model_class>
def optimize_predictions(preds, coeffs):
preds[preds <= coeffs[0]] = 0
preds[np.where(np.logical_and(preds > coeffs[0], preds <= coeffs[1])) ] = 1
preds[np.where(np.logical_and(preds > coeffs[1], preds <= coeffs[2])) ] = 2
preds[np.where(np.logical_and(preds > coeffs[2], preds <= coeffs[3])) ] = 3
preds[np.where(np.logical_and(preds > coeffs[3], preds <= coeffs[4])) ] = 4
preds[np.where(np.logical_and(preds > coeffs[4], preds <= coeffs[5])) ] = 5
preds[np.where(np.logical_and(preds > coeffs[5], preds <= coeffs[6])) ] = 6
preds[np.where(np.logical_and(preds > coeffs[6], preds <= coeffs[7])) ] = 7
preds[np.where(np.logical_and(preds > coeffs[7], preds <= coeffs[8])) ] = 8
preds[np.where(np.logical_and(preds > coeffs[8], preds <= coeffs[9])) ] = 9
preds[preds > coeffs[9]] = 10
preds = preds.astype(np.int8)
 return preds
<choose_model_class><EOS> 
test['open_channels'] = np.round(np.clip(final_preds, 0, 10)).astype(int)
 test[['time','open_channels']].to_csv('submission.csv', index=False, float_format='%.4f' )