Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<load_from_csv>
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 %matplotlib inline
sample_sub = pd.read_csv("/kaggle/input/Kannada-MNIST/sample_submission.csv" )<prepare_x_and_y>
BATCHSIZE = 50000
SEED = 529
SELECT = True
SPLITS = 5
fe_config = [ 
(True, True, 50000, None), 
(False, False, 5000, None),
]
X=train.iloc[:,1:].values
Y=train.iloc[:,0].values
Y[:10]<split>
def seed_everything(seed : int)-> NoReturn :
rn.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 seed_everything(SEED)
X_train, X_valid, Y_train, Y_valid = train_test_split(X, Y, test_size = 0.10, random_state=42 )<choose_model_class>
def read_data(base : os.path.abspath)-> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
train = pd.read_csv(os.path.join(base + '/train.csv'), dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int8})
 test = pd.read_csv(os.path.join(base + '/test.csv'), dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv(os.path.join(base + '/sample_submission.csv'), dtype={'time': np.float32})
 return train, test, sub
train_datagen = ImageDataGenerator(rescale = 1./255.,
rotation_range = 10,
width_shift_range = 0.25,
height_shift_range = 0.25,
shear_range = 0.1,
zoom_range = 0.25,
horizontal_flip = False )<define_variables>
def batching(df : pd.DataFrame,
batch_size : int,
add_index : Optional[bool]=True)-> pd.DataFrame :
df['batch_'+ str(batch_size)] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values + 1
df['batch_'+ str(batch_size)] = df['batch_'+ str(batch_size)].astype(np.uint16)
 if add_index:
df['batch_' + str(batch_size)+'_idx'] = df.index -(df['batch_'+ str(batch_size)] * batch_size)
 df['batch_' + str(batch_size)+'_idx'] = df['batch_' + str(batch_size)+'_idx'].astype(np.uint16)
 return df
valid_datagen = ImageDataGenerator(rescale=1./255 )<normalization>
def maddest(d : Union[np.array, pd.Series, List], axis : Optional[int]=None)-> np.array:
return np.mean(np.absolute(d - np.mean(d, axis)) , axis)
def lr_decay(epoch):
return initial_learningrate * 0.99 ** epoch<choose_model_class>
def denoise_signal(x : Union[np.array, pd.Series],
wavelet : Optional[Text]='db4',
level : Optional[int]=1)-> np.array:
coeff = pywt.wavedec(x, wavelet, mode="per")
 sigma =(1/0.6745)* maddest(coeff[-level])
 uthresh = sigma * np.sqrt(2*np.log(len(x)))
 coeff[1:] =(pywt.threshold(i, value=uthresh, mode='hard')for i in coeff[1:])
 return pywt.waverec(coeff, wavelet, mode='per')
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(64,(3,3), padding='same', input_shape=(28, 28, 1)) ,
tf.keras.layers.BatchNormalization(momentum=0.5, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(64,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2, 2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.2, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(256,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.2, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(256,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(256),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(10, activation='softmax')
 ])
 model.summary()<define_variables>
def denoise_signal_simple(x : Union[np.array, pd.Series],
wavelet : Optional[Text]='db4',
level : Optional[int]=1)-> np.array:
coeff = pywt.wavedec(x, wavelet, mode="per")
 uthresh = 10
coeff[1:] =(pywt.threshold(i, value=uthresh, mode='hard')for i in coeff[1:])
 return pywt.waverec(coeff, wavelet, mode='per')
initial_learningrate=2e-3
batch_size = 1024
epochs = 50
input_shape =(28, 28, 1 )<choose_model_class>
def trend(df : Union[pd.Series, np.array],
abs_values: Optional[bool]=False)-> float:
idx = np.array(range(len(df)))
 if abs_values:
df = np.abs(df)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), df)
 return lr.coef_[0]
es = EarlyStopping(monitor='val_loss', verbose=1, patience=10 )<choose_model_class>
def change_rate(df : Union[pd.Series, np.array])-> float:
change =(np.diff(df)/ df[:-1])
 change = change[np.nonzero(change)[0]]
change = change[~np.isnan(change)]
change = change[change != -np.inf]
change = change[change != np.inf]
return np.mean(change)
model.compile(loss="categorical_crossentropy",
optimizer=RMSprop(lr=initial_learningrate),
metrics=['accuracy'] )<import_modules>
def lag_with_pct_change(df : pd.DataFrame,
batch_size : int,
shift_sizes : Optional[List]=[1, 2],
add_pct_change : Optional[bool]=False,
add_pct_change_lag : Optional[bool]=False)-> pd.DataFrame:
assert 'batch_' + str(batch_size)+'_idx' in df.columns
for shift_size in shift_sizes:
df['signal_shift_pos_'+str(shift_size)] = df['signal'].shift(shift_size ).fillna(0)
 df['signal_shift_neg_'+str(shift_size)] = df['signal'].shift(-1*shift_size ).fillna(0)
 for i in df[df['batch_' + str(batch_size)+'_idx'].isin(range(shift_size)) ].index:
df['signal_shift_pos_'+str(shift_size)][i] = np.nan
for i in df[df['batch_' + str(batch_size)+'_idx'].isin(range(batch_size - shift_size, batch_size)) ].index:
df['signal_shift_neg_'+str(shift_size)][i] = np.nan
if add_pct_change:
df['pct_change'] = df['signal'].pct_change()
if add_pct_change_lag:
df['pct_change_shift_pos_'+str(shift_size)] = df['pct_change'].shift(shift_size ).fillna(0)
 df['pct_change_shift_neg_'+str(shift_size)] = df['pct_change'].shift(-1*shift_size ).fillna(0)
 for i in df[df['batch_' + str(batch_size)+'_idx'].isin(range(shift_size)) ].index:
df['pct_change_shift_pos_'+str(shift_size)][i] = np.nan
for i in df[df['batch_' + str(batch_size)+'_idx'].isin(range(batch_size - shift_size, batch_size)) ].index:
df['pct_change_shift_neg_'+str(shift_size)][i] = np.nan
return df
from sklearn import metrics 
<train_model>
def feature_enginering_by_batch(z : Union[pd.Series, np.array],
batch_size : int,
window_size : Optional[List]=None)-> pd.DataFrame:
temp = pd.DataFrame(index=[0], dtype=np.float16)
 temp['mean'] = z.mean()
temp['max'] = z.max()
temp['min'] = z.min()
temp['std'] = z.std()
temp['mean_abs_chg'] = np.mean(np.abs(np.diff(z)))
 temp['abs_max'] = np.max(np.abs(z))
temp['abs_min'] = np.min(np.abs(z))
temp['range'] = temp['max'] - temp['min']
temp['max_to_min'] = temp['max'] / temp['min']
temp['abs_avg'] =(temp['abs_max'] + temp['abs_min'])/ 2
for i in range(1, 5):
temp[f'kstat_{i}'] = stats.kstat(z, i)
 for i in range(2, 5):
temp[f'moment_{i}'] = stats.moment(z, i)
 for i in [1, 2]:
temp[f'kstatvar_{i}'] = stats.kstatvar(z, i)
 if window_size is not None:
for window in window_size:
temp['percentile_roll_'+str(window)+'_std_25'] = np.percentile(pd.Series(z ).rolling(window ).std().dropna().values, 25)
 temp['percentile_roll_'+str(window)+'_std_75'] = np.percentile(pd.Series(z ).rolling(window ).std().dropna().values, 75)
 temp['percentile_roll_'+str(window)+'_std_05'] = np.percentile(pd.Series(z ).rolling(window ).std().dropna().values, 5)
 temp['percentile_roll_'+str(window)+'_std_95'] = np.percentile(pd.Series(z ).rolling(window ).std().dropna().values, 95)
 temp['percentile_roll_'+str(window)+'_mean_25'] = np.percentile(pd.Series(z ).rolling(window ).mean().dropna().values, 25)
 temp['percentile_roll_'+str(window)+'_mean_75'] = np.percentile(pd.Series(z ).rolling(window ).mean().dropna().values, 75)
 temp['percentile_roll_'+str(window)+'_mean_05'] = np.percentile(pd.Series(z ).rolling(window ).mean().dropna().values, 5)
 temp['percentile_roll_'+str(window)+'_mean_95'] = np.percentile(pd.Series(z ).rolling(window ).mean().dropna().values, 95)
 return temp
history = model.fit_generator(
 train_datagen.flow(X_train,Y_train, batch_size=batch_size),
steps_per_epoch=100,
epochs=epochs,
callbacks=[LearningRateScheduler(lr_decay),
es
],
validation_data=valid_datagen.flow(X_valid,Y_valid),
validation_steps=50,
verbose=2 )<compute_train_metric>
def parse_sample(sample : pd.DataFrame,
batch_no : int,
batch_size : int,
window_size : List)-> pd.DataFrame:
temp = feature_enginering_by_batch(sample['signal'].values, batch_size, window_size)
 temp['batch_'+ str(batch_size)] = int(batch_no)
 return temp
preds_dig=model.predict_classes(x_dig/255)
 metrics.accuracy_score(preds_dig, y_dig )<drop_column>
def sample_gen(df : pd.DataFrame,
batch_size : int,
window_size : List,
batches : List=[0],)-> pd.DataFrame:
result = Parallel(n_jobs=1, temp_folder='/tmp', max_nbytes=None, backend='multiprocessing' )(delayed(parse_sample)
(df[df['batch_'+ str(batch_size)]==i], int(i), batch_size, window_size)
 for i in tqdm(batches))
data = [r.values for r in result]
data = np.vstack(data)
 cols = result[0].columns
cols = [name+'_'+str(batch_size)if name!='batch_'+ str(batch_size)else 'batch_'+ str(batch_size)for name in cols ]
X = pd.DataFrame(data, columns=cols)
 X = reduce_mem_usage(X, False)
 X = X.sort_values('batch_'+ str(batch_size))
return X
keras.backend.clear_session()<choose_model_class>
def run_feat_enginnering(df : pd.DataFrame,
create_all_data_feats : bool,
add_index : bool,
batch_size : int,
window_size : List)-> pd.DataFrame:
df = batching(df, batch_size=batch_size, add_index=add_index)
 if create_all_data_feats:
df = lag_with_pct_change(df, batch_size, [1, 2, 4], add_pct_change=True, add_pct_change_lag=True)
 batches = df['batch_'+ str(batch_size)].unique().tolist()
batch_feats=sample_gen(df, batch_size=batch_size, window_size=window_size, batches=batches)
 df = pd.merge(df, batch_feats, on='batch_'+ str(batch_size), how='left')
 df = reduce_mem_usage(df, False)
 return df
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(64,(3,3), padding='same', input_shape=(28, 28, 1)) ,
tf.keras.layers.BatchNormalization(momentum=0.5, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(64,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2, 2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.2, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(256,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.2, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(256,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(256),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(10, activation='softmax')
 ])
<define_variables>
def feature_selection(df : pd.DataFrame,
df_test : pd.DataFrame,
subtract_only : Optional[bool]=True,
idx_cols : List=['time'],
target_col : List=['open_channels'])-> Tuple[pd.DataFrame , pd.DataFrame]:
drops = df.columns[df.isna().sum() >25000]
df = df.drop(drops, axis=1)
 df = df.replace([np.inf, -np.inf], np.nan)
 df = df.fillna(0)
 gc.collect()
if subtract_only == False:
corrcoef_cols = [col for col in df.columns.tolist() if col not in(idx_cols+target_col)]
first=dict() ; second=dict() ; third=dict()
for col in corrcoef_cols:
ss = np.corrcoef(df[col], df['open_channels'])[0, 1]
first[col] = ss
ss = np.corrcoef(df[col]-df['signal'], df['open_channels'])[0, 1]
second[col] = ss
ss = np.corrcoef(df[col]*df['signal'], df['open_channels'])[0, 1]
third[col] = ss
corr_df = pd.DataFrame.from_dict(
 {
'Base':first,
'Signal-Subtracted': second,
'Signal-Multiplied': third
}
).fillna(0 ).apply(np.abs ).sort_values('Base', ascending=False)
 base_cols = corr_df.sort_values('Base', ascending=False ).head(100 ).index.tolist()
multiply_cols = corr_df.sort_values('Signal-Multiplied', ascending=False ).head(10 ).index.tolist()
subtract_cols = corr_df.sort_values('Signal-Subtracted', ascending=False ).head(25 ).index.tolist()
display(corr_df.sort_values('Base', ascending=False ).tail(50))
all_cols = list(set(base_cols + multiply_cols + subtract_cols + idx_cols + target_col))
all_cols_test = list(set(base_cols + multiply_cols + subtract_cols + idx_cols))
drops = list(set(multiply_cols + subtract_cols)-set(base_cols))
df = df[all_cols]
df_test = df_test[all_cols_test]
for col in multiply_cols:
df[col+'_m'] = df[col] * df['signal']
df_test[col+'_m'] = df_test[col] * df_test['signal']
for col in subtract_cols:
df[col+'_s'] = df[col] - df['signal']
df_test[col+'_s'] = df_test[col] - df_test['signal']
df = df.drop(drops, axis=1)
 else:
not_imp = ['kstat_1_5000', 'kstat_2_5000', 'kstat_3_5000', 'kstat_4_5000', 'moment_2_5000',
'moment_3_5000','moment_4_5000', 'kstatvar_1_5000', 'kstatvar_2_5000','kstat_1_50000',
'kstat_2_50000', 'kstat_3_50000', 'kstat_4_50000', 'moment_2_50000', 'moment_3_50000',
'moment_4_50000', 'kstatvar_1_50000', 'kstatvar_2_50000']
subtract_cols = list(set(df.columns.tolist())-set(idx_cols + target_col + not_imp))
for col in subtract_cols:
df[col+'_s'] = df[col] - df['signal']
df_test[col+'_s'] = df_test[col] - df_test['signal']
df = reduce_mem_usage(df, False)
 df_test = reduce_mem_usage(df_test, False)
 gc.collect()
return df, df_test
initial_learningrate=2e-3
batch_size = 1024
epochs = 50
input_shape =(28, 28, 1 )<choose_model_class>
def MacroF1Metric(preds : np.array, dtrain : lgb.Dataset)-> Tuple[Text, np.array, bool] :
labels = dtrain.get_label()
preds = np.round(np.clip(preds, 0, 10)).astype(int)
 score = f1_score(labels, preds, average = 'macro')
 return('MacroF1Metric', score, True)
model.compile(loss="categorical_crossentropy",
optimizer=RMSprop(lr=initial_learningrate),
metrics=['accuracy'] )<train_model>
def run_cv_model_by_batch(train : pd.DataFrame,
test : pd.DataFrame,
splits : int,
shuffle : bool,
seed : int,
batch_col : Text,
params : Dict,
feats : List,
sample_submission: pd.DataFrame)-> pd.DataFrame:
oof_ = np.zeros(len(train))
preds_ = np.zeros(len(test))
target = ['open_channels']
imp_df = pd.DataFrame(index=feats)
 kf = KFold(splits, shuffle, seed)
 for n_fold,(tr_idx, val_idx)in enumerate(kf.split(train, train[target], groups=train[batch_col])) :
trn_data = lgb.Dataset(train[feats].iloc[tr_idx], label=train[target].iloc[tr_idx])
 val_data = lgb.Dataset(train[feats].iloc[val_idx], label=train[target].iloc[val_idx])
 model = lgb.train(params, trn_data, 10000, valid_sets = [trn_data, val_data], verbose_eval=0 , early_stopping_rounds = 50)
 oof_[val_idx] += model.predict(train[feats].iloc[val_idx], num_iteration=model.best_iteration)
 preds_ += model.predict(test[feats], num_iteration=model.best_iteration)/ SPLITS
f1_score_ = f1_score(train[target].iloc[val_idx], np.round(np.clip(oof_[val_idx], 0, 10)).astype(int), average = 'macro')
 rmse_score_ = np.sqrt(mean_squared_error(train[target].iloc[val_idx], oof_[val_idx]))
logger.info(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f} rmse score : {rmse_score_:1.5f}')
 imp_df[f'feat_importance_{n_fold + 1}'] = model.feature_importance(importance_type='gain')
 f1_score_ = f1_score(train[target], np.round(np.clip(oof_, 0, 10)).astype(int), average = 'macro')
 rmse_score_ = np.sqrt(mean_squared_error(train[target], oof_))
logger.info(f'Training completed.oof macro f1 score : {f1_score_:1.5f} oof rmse score : {rmse_score_:1.5f}')
 sample_submission['open_channels'] = np.round(np.clip(preds_, 0, 10)).astype(int)
 sample_submission.to_csv('submission.csv', index=False, float_format='%.4f')
 display(sample_submission.head())
 np.save('oof.npy', oof_)
 np.save('preds.npy', preds_)
 return imp_df
history = model.fit_generator(
 train_datagen.flow(X,Y, batch_size=batch_size),
steps_per_epoch=100,
epochs=epochs,
callbacks=[LearningRateScheduler(lr_decay)
 ],
validation_data=valid_datagen.flow(X_valid,Y_valid),
validation_steps=50,
verbose=2 )<compute_train_metric>
def get_params(seed : int)-> Dict :
params = dict()
params['learning_rate']=0.009;
params['max_depth']=-1;
params['num_leaves']=257;
params['metric']='rmse';
params['random_state']=seed;
params['n_jobs']=-1;
params['feature_fraction']=1 ;
params['boosting']='goss';
params['boost_from_average']=True;
params['bagging_seed']=seed;
params['bagging_freq']=0;
params['bagging_fraction']=1;
params['reg_alpha']=0;
params['reg_lambda']=0
params['force_row_wise']=True
return params
<predict_on_test><EOS> 
def run_everything(fe_config : List)-> NoReturn:
not_feats_cols = ['time']
target_col = ['open_channels']
init_logger()
with timer(f'Reading Data'):
logger.info('Reading Data Started...')
 base = os.path.abspath('/kaggle/input/liverpool-ion-switching/')
 train, test, sample_submission = read_data(base)
 logger.info('Reading Data Completed...')
 with timer(f'Creating Features'):
logger.info('Feature Enginnering Started...')
 for config in fe_config:
train = run_feat_enginnering(train, create_all_data_feats=config[0], add_index=config[1], batch_size=config[2], window_size=config[3])
 test = run_feat_enginnering(test, create_all_data_feats=config[0], add_index=config[1], batch_size=config[2], window_size=config[3])
 not_feats_cols.append('batch_'+str(config[2]))
if config[1]:
not_feats_cols.append('batch_'+str(config[2])+'_idx')
 if SELECT:
train, test = feature_selection(train, test, subtract_only=True, idx_cols=not_feats_cols, target_col=target_col)
 logger.info('Feature Enginnering Completed...')
 with timer(f'Running LGB model'):
logger.info(f'Training LGB model with {SPLITS} folds Started...')
 params = get_params(SEED)
 feats = [c for c in train.columns if c not in(not_feats_cols+target_col)]
importances = run_cv_model_by_batch(train, test, splits=SPLITS, shuffle=True, seed=SEED, batch_col='batch_50000',params=params, feats=feats, sample_submission=sample_submission)
 importances.to_csv('importances.csv')
 logger.info(f'Training completed...')