Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<split>
!pip install tensorflow_addons
warnings.simplefilter('ignore')
 warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
def objective_lgb(trial,data=X,target=y):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'tree_method':'gpu_hist',
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-3, 1),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-3, 1),
'feature_framcion': trial.suggest_categorical('feature_framcion', [0.1, 0.2, 0.3,0.5,0.7,0.9]),
'bagging_fraction': trial.suggest_categorical('bagging_framcion', [0.1, 0.2,0.3,0.4,0.5,0.8,1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.0008, 0.01, 0.015, 0.02,0.03, 0.05,0.08,0.1]),
'n_estimators': 4000,
'num_leaves': trial.suggest_categorical('num_leaves', [31,50,150,200,250,300,350]),
'max_depth': trial.suggest_categorical('max_depth', [5,7,9,11,13,15,17,20,23,25]),
'min_data_in_leaf': trial.suggest_categorical('min_data_in_leaf', [10,20,30]),
'min_child_weight': trial.suggest_categorical('min_child_weight', [0.001,0.005, 0.01, 0.05, 0.1,0.5]),
'random_state': 48
}
model = lgb.LGBMRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse<train_model>
train = pd.read_csv('.. /input/liverpool-ion-switching/train.csv')
 test = pd.read_csv('.. /input/liverpool-ion-switching/test.csv')
 ss = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv' )
study = optuna.create_study(direction='minimize')
 study.optimize(objective_lgb, n_trials=50)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )<find_best_params>
def make_batches(df, dataset="train"):
batches = []
batch_size = [500000, 100000]
if dataset == "train":
for idx in range(10):
batches.append(df[idx * batch_size[0]:(idx + 1)* batch_size[0]])
 else:
for idx in range(10):
batches.append(df[idx * batch_size[1]:(idx + 1)* batch_size[1]])
 for idx in range(2):
base = 10 * batch_size[1]
batches.append(df[base + idx * batch_size[0]: base +(idx + 1)* batch_size[0]])
 return batches
train_batches = make_batches(train, "train")
 test_batches = make_batches(test, "test" )
study.best_trial.params<init_hyperparams>
linear_train_idx = [1]
linear_test_idx = [0, 1, 4, 6, 7, 8]
def linear_drift(x, x0):
return 0.3 *(x - x0)
 def remove_linear_drift(data, dataset="train"):
if dataset == "train":
data[1].loc[data[1].index[0:100000], 'signal'] = data[1].signal[0:100000].values - linear_drift(data[1].time[0:100000].values, data[1].time[0:1].values)
 else:
for idx in linear_test_idx:
data[idx].loc[data[idx].index[0:100000], 'signal'] = data[idx].signal[0:100000].values - linear_drift(data[idx].time[0:100000].values, data[idx].time[0:1].values)
 return data
train_drift_removed = remove_linear_drift(train_batches, "train")
 test_drift_removed = remove_linear_drift(test_batches, "test" )
Best_params_lgb = {'lambda_l2': 0.013616569506899653,
'lambda_l1': 0.006495842188985166,
'feature_framcion': 0.3,
'bagging_framcion': 0.3,
'learning_rate': 0.015,
'num_leaves': 200,
'max_depth': 25,
'min_data_in_leaf': 30,
'min_child_weight': 0.001,
'n_estimators': 3000,
'random_state': 48,
'tree_method':'gpu_hist'}<train_model>
df_train = train_drift_removed.copy()
df_test = test_drift_removed.copy()
train_x, test_x, train_y, test_y = train_test_split(X, y, test_size=0.15,random_state=42)
 model_lgb = lgb.LGBMRegressor(**Best_params_lgb)
 model_lgb.fit(train_x,train_y,eval_set=[(test_x,test_y)],
early_stopping_rounds=100,verbose=False )<compute_train_metric>
def my_sin(x, A, ph, d):
frequency = 0.01
omega = 2 * np.pi * frequency
return A * np.sin(omega * x + ph)+ d
def parabolic_drift_fit(data):
x = data['time']
y = data['signal']
frequency = 0.01
omega = 2 * np.pi * frequency
M = np.array([[np.sin(omega * t), np.cos(omega * t), 1] for t in x])
 y = np.array(y ).reshape(len(y), 1)
(theta, _, _, _)= np.linalg.lstsq(M, y)
 A = np.sqrt(theta[0,0]**2 + theta[1,0]**2)
 ph = math.atan2(theta[1,0], theta[0,0])
 d = theta[2,0]
popt = [A, ph, d]
print(popt)
 return popt
parabola_params = []
for idx in parabola_train_idx:
parabola_params.append(parabolic_drift_fit(df_train[idx]))
parabola_params.append(parabolic_drift_fit(df_test[parabola_test_idx[0]]))
plt.figure(figsize=(30, 4))
for n, idx in enumerate(parabola_train_idx):
plt.subplot("15" + str(n + 1))
plt.title("Train " + str(idx))
plt.ylabel("Signal", fontsize=8)
 plt.plot(df_train[idx]['time'], df_train[idx]['signal'], linewidth=.1)
 plt.plot(df_train[idx]['time'], my_sin(df_train[idx]['time'], *parabola_params[n]), 'y')
 plt.grid()
plt.ylim([np.min(df_train[idx]['signal']), np.min(df_train[idx]['signal'])+ 18])
 plt.subplot("155")
 plt.title("Test 10")
 plt.ylabel("Signal", fontsize=8)
 plt.ylim([np.min(df_test[10]['signal']), np.min(df_test[10]['signal'])+ 18])
 plt.plot(df_test[10]['time'], df_test[10]['signal'], linewidth=.1)
 plt.plot(df_test[10]['time'], my_sin(df_test[10]['time'], *parabola_params[-1]), 'y')
 plt.grid()
preds = model_lgb.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 rmse<prepare_x_and_y>
def my_sin(x, A, ph, d):
frequency = 0.01
omega = 2 * np.pi * frequency
return A * np.sin(omega * x + ph)+ d
def parabolic_drift_fit(data):
x = data['time']
y = data['signal']
frequency = 0.01
omega = 2 * np.pi * frequency
M = np.array([[np.sin(omega * t), np.cos(omega * t), 1] for t in x])
 y = np.array(y ).reshape(len(y), 1)
(theta, _, _, _)= np.linalg.lstsq(M, y)
 A = np.sqrt(theta[0,0]**2 + theta[1,0]**2)
 ph = math.atan2(theta[1,0], theta[0,0])
 d = theta[2,0]
popt = [A, ph, d]
print(popt)
 return popt
parabola_params = []
for idx in parabola_train_idx:
parabola_params.append(parabolic_drift_fit(df_train[idx]))
parabola_params.append(parabolic_drift_fit(df_test[parabola_test_idx[0]]))
plt.figure(figsize=(30, 4))
for n, idx in enumerate(parabola_train_idx):
plt.subplot("15" + str(n + 1))
plt.title("Train " + str(idx))
plt.ylabel("Signal", fontsize=8)
 plt.plot(df_train[idx]['time'], df_train[idx]['signal'], linewidth=.1)
 plt.plot(df_train[idx]['time'], my_sin(df_train[idx]['time'], *parabola_params[n]), 'y')
 plt.grid()
plt.ylim([np.min(df_train[idx]['signal']), np.min(df_train[idx]['signal'])+ 18])
 plt.subplot("155")
 plt.title("Test 10")
 plt.ylabel("Signal", fontsize=8)
 plt.ylim([np.min(df_test[10]['signal']), np.min(df_test[10]['signal'])+ 18])
 plt.plot(df_test[10]['time'], df_test[10]['signal'], linewidth=.1)
 plt.plot(df_test[10]['time'], my_sin(df_test[10]['time'], *parabola_params[-1]), 'y')
 plt.grid()
test_X = test[features]<predict_on_test>
parabola_train_idx = [6, 7, 8, 9]
parabola_test_idx = [10]
def parabolic_drift(x, t=0):
f = 0.01
omega = 2 * np.pi * f
return 5 * np.sin(omega * x + t * np.pi)
 def remove_parabolic_drift(data, dataset="train"):
if dataset == "train":
for idx in parabola_train_idx:
data[idx].loc[data[idx].index[0:500000], 'signal'] = data[idx].signal[0:500000].values - parabolic_drift(data[idx].time[0:500000].values,(idx % 2))
else:
data[10].loc[data[10].index[0:500000], 'signal'] = data[10].signal[0:500000].values - parabolic_drift(data[10].time[0:500000].values)
 return data
df_train = remove_parabolic_drift(train_drift_removed, "train")
 df_test = remove_parabolic_drift(test_drift_removed, "test" )
preds = model_lgb.predict(test_X )<save_to_csv>
df_train_clean = df_train[0]
df_test_clean = df_test[0]
for df in df_train[1:]:
df_train_clean = pd.concat([df_train_clean, df], ignore_index=True)
 for df in df_test[1:]:
df_test_clean = pd.concat([df_test_clean, df], ignore_index=True)
 df_train_clean.to_csv("train_wo_drift.csv", index=False, float_format="%.4f")
 df_test_clean.to_csv("test_wo_drift.csv", index=False, float_format="%.4f" )
sub['target']=preds
sub.to_csv('submission_lgb.csv', index=False )<load_from_csv>
def read_data() :
train = pd.read_csv('/kaggle/working/train_wo_drift.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/working/test_wo_drift.csv', dtype={'time': np.float32, 'signal': np.float32})
 sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 return train, test, sub
train = pd.read_csv(input_path / 'train.csv')
 display(train.head() )<load_from_csv>
read_data()
test = pd.read_csv(input_path / 'test.csv')
 display(test.head() )<load_from_csv>
EPOCHS = 100
NNBATCHSIZE = 32
GROUP_BATCH_SIZE = 4000
SEED = 123
LR = 0.001
SPLITS = 4
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )
<set_options><EOS> 
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features
def Classifier(shape_):
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = Conv1D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = Conv1D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 return res_x
inp = Input(shape =(shape_))
x = wave_block(inp, 32, 3, 12)
 x = wave_block(x, 64, 3, 8)
 x = wave_block(x, 64, 3, 4)
 x = wave_block(x, 128, 3, 1)
 out = Dense(11, activation = 'softmax', name = 'out' )(x)
 model = models.Model(inputs = inp, outputs = out)
 opt = Adam(lr = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(loss = losses.CategoricalCrossentropy() , optimizer = opt, metrics = ['accuracy'])
 return model
def lr_schedule(epoch):
if epoch < 30:
lr = LR
elif epoch < 40:
lr = LR / 3
elif epoch < 50:
lr = LR / 5
elif epoch < 60:
lr = LR / 7
elif epoch < 70:
lr = LR / 9
elif epoch < 80:
lr = LR / 11
elif epoch < 90:
lr = LR / 13
else:
lr = LR / 100
return lr
class MacroF1(Callback):
def __init__(self, model, inputs, targets):
self.model = model
self.inputs = inputs
self.targets = np.argmax(targets, axis = 2 ).reshape(-1)
 def on_epoch_end(self, epoch, logs):
pred = np.argmax(self.model.predict(self.inputs), axis = 2 ).reshape(-1)
 score = f1_score(self.targets, pred, average = 'macro')
 print(f'F1 Macro Score: {score:.5f}')
 def run_cv_model_by_batch(train, test, splits, batch_col, feats, sample_submission, nn_epochs, nn_batch_size):
seed_everything(SEED)
 K.clear_session()
config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=config)
 tf.compat.v1.keras.backend.set_session(sess)
 oof_ = np.zeros(( len(train), 11))
preds_ = np.zeros(( len(test), 11))
target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=4)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 tr = pd.concat([pd.get_dummies(train.open_channels), train[['group']]], axis=1)
 tr.columns = ['target_'+str(i)for i in range(11)] + ['group']
target_cols = ['target_'+str(i)for i in range(11)]
train_tr = np.array(list(tr.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[feats].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[feats].values)))
 for n_fold,(tr_idx, val_idx, val_orig_idx)in enumerate(new_splits[0:], start=0):
train_x, train_y = train[tr_idx], train_tr[tr_idx]
valid_x, valid_y = train[val_idx], train_tr[val_idx]
print(f'Our training dataset shape is {train_x.shape}')
 print(f'Our validation dataset shape is {valid_x.shape}')
 gc.collect()
shape_ =(None, train_x.shape[2])
 model = Classifier(shape_)
 cb_lr_schedule = LearningRateScheduler(lr_schedule)
 model.fit(train_x,train_y,
epochs = nn_epochs,
callbacks = [cb_lr_schedule, MacroF1(model, valid_x, valid_y)],
batch_size = nn_batch_size,verbose = 2,
validation_data =(valid_x,valid_y))
preds_f = model.predict(valid_x)
 f1_score_ = f1_score(np.argmax(valid_y, axis=2 ).reshape(-1), np.argmax(preds_f, axis=2 ).reshape(-1), average = 'macro')
 print(f'Training fold {n_fold + 1} completed.macro f1 score : {f1_score_ :1.5f}')
 preds_f = preds_f.reshape(-1, preds_f.shape[-1])
 oof_[val_orig_idx,:] += preds_f
te_preds = model.predict(test)
 te_preds = te_preds.reshape(-1, te_preds.shape[-1])
 preds_ += te_preds / SPLITS
f1_score_ = f1_score(np.argmax(train_tr, axis = 2 ).reshape(-1), np.argmax(oof_, axis = 1), average = 'macro')
 print(f'Training completed.oof macro f1 score : {f1_score_:1.5f}')
 sample_submission['open_channels'] = np.argmax(preds_, axis = 1 ).astype(int)
 sample_submission.to_csv('submission_wavenet.csv', index=False, float_format='%.4f')
 def run_everything() :
print('Reading Data Started...')
 train, test, sample_submission = read_data()
train, test = normalize(train, test)
 print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = run_feat_engineering(train, batch_size = GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size = GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print('Feature Engineering Completed...')
 print(f'Training Wavenet model with {SPLITS} folds of GroupKFold Started...')
 run_cv_model_by_batch(train, test, SPLITS, 'group', features, sample_submission, EPOCHS, NNBATCHSIZE)
 print('Training completed...')
 run_everything()