Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<define_variables>
df = pd.read_csv("/kaggle/input/remove-trends-giba/train_clean_giba.csv", usecols=["signal","open_channels"], dtype={'signal': np.float32, 'open_channels':np.int32})
 test_df = pd.read_csv("/kaggle/input/remove-trends-giba/test_clean_giba.csv", usecols=["signal"], dtype={'signal': np.float32})
 df.shape, test_df.shape
param_name = "alpha"<define_variables>
df['group'] = np.arange(df.shape[0])//500_000
aug_df = df[df["group"] == 5].copy()
aug_df["group"] = 10
for col in ["signal", "open_channels"]:
aug_df[col] += df[df["group"] == 8][col].values
df = df.append(aug_df, sort=False ).reset_index(drop=True)
 df.shape
del aug_df
gc.collect()
param_name = "lambda"<define_search_space>
wavenet_oof =(np.load("/kaggle/input/into-the-wild-wavenet/wavenet.npz")["valid"] + np.load("/kaggle/input/into-the-wild-wavenet/wavenet.npz")["tta"])/2
wavenet_test = np.load("/kaggle/input/into-the-wild-wavenet/wavenet.npz")["test"]
for i in range(wavenet_oof.shape[1]):
df["prob_{}".format(i)] = wavenet_oof[:, i]
test_df["prob_{}".format(i)] = wavenet_test[:, i]
df["wave_pred"] = np.argmax(wavenet_oof, axis=1)
 test_df["wave_pred"] = np.argmax(wavenet_test, axis=1)
 df['noise'] = df['signal'].values - df['wave_pred'].values
test_df['noise'] = test_df['signal'].values - test_df['wave_pred'].values
param_range = np.linspace(0.1, 1, 10)
 param_range<define_variables>
def get_margin(x):
return np.log(x)
 M = get_margin(wavenet_oof)
 M_test = get_margin(wavenet_test )
param_name = 'colsample_bytree'<define_variables>
f1_score(df["open_channels"], df["wave_pred"], average="macro" )
param_name = 'subsample'<define_search_space>
log_loss(df["open_channels"], wavenet_oof )
param_name = 'n_estimators'<define_search_space>
NUM_FOLDS = 5
df["mg"] = df.index//100_000
test_df["mg"] = test_df.index//100_000
df["fold"] = df["mg"] % NUM_FOLDS
param_range = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]<prepare_x_and_y>
for data in [df, test_df]:
y_time_since = np.empty(( data.shape[0], 11))
y_time_till = np.empty(( data.shape[0], 11))
y_pred = data["wave_pred"].values
for sec in range(data.shape[0]//100_000):
begin, end = sec*100_000,(sec+1)*100_000
last_seen = np.array([np.nan]*11)
 for index in range(begin, end):
y_time_since[index] = index - last_seen
last_seen[y_pred[index]] = index
last_seen = np.array([np.nan]*11)
 for index in reversed(range(begin, end)) :
y_time_till[index] = last_seen - index
last_seen[y_pred[index]] = index
for i in range(11):
f = "time_since_{}".format(i)
 data[f] = y_time_since[:, i]
data[f] = np.clip(data[f].fillna(np.inf), 0, 1_000)
 f = "time_till_{}".format(i)
 data[f] = y_time_till[:, i]
data[f] = np.clip(data[f].fillna(np.inf), 0, 1_000)
 test_df.sample(2 ).T
X = train[features]
y = train['target']<split>
features = ["signal", "noise",
"prob_0", "prob_1", "prob_2", "prob_3", "prob_4", "prob_5", "prob_6", "prob_7", "prob_8", "prob_9", "prob_10"]
for i in range(11):
f = "time_since_{}".format(i)
 features.append(f)
 f = "time_till_{}".format(i)
 features.append(f )
def objective(trial,data=X,target=y):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'tree_method':'gpu_hist',
'lambda': trial.suggest_loguniform('lambda', 1e-3, 1),
'alpha': trial.suggest_loguniform('alpha', 1e-3, 1),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.1, 0.2, 0.3,0.5,0.7,0.9]),
'subsample': trial.suggest_categorical('subsample', [0.1, 0.2,0.3,0.4,0.5,0.8,1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.0008, 0.01, 0.015, 0.02,0.03, 0.05,0.08,0.1]),
'n_estimators': 4000,
'max_depth': trial.suggest_categorical('max_depth', [5,7,9,11,13,15,17,20,23,25]),
'random_state': 48,
'min_child_weight': trial.suggest_int('min_child_weight', 1, 400),
}
model = xgb.XGBRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse<train_model>
params = {"objective": "multi:softprob",
"num_class": 11,
"learning_rate" : 0.2,
"max_leaves": 2**4,
"grow_policy": "lossguide",
'min_child_weight': 50,
'lambda': 2,
'eval_metric': 'mlogloss',
"base_score": 0,
"tree_method": 'gpu_hist', "gpu_id": 0
}
target = "open_channels"
y_oof = np.zeros_like(wavenet_oof)
 y_test = np.zeros_like(wavenet_test)
 del wavenet_oof, wavenet_test
gc.collect()
X_test = test_df[features].values
d_test = xgb.DMatrix(X_test)
 d_test.set_base_margin(M_test.flatten())
 for f in range(NUM_FOLDS):
train_df, val_df = df[df["fold"] != f].copy() , df[df["fold"] == f].copy()
train_ind = np.where(df["fold"].values != f)[0]
val_ind = np.where(df["fold"].values == f)[0]
X_train, X_val = train_df[features].values, val_df[features].values
y_train, y_val = train_df[target].values, val_df[target].values
d_train = xgb.DMatrix(X_train, y_train)
 d_train.set_base_margin(M[train_ind].flatten())
 d_val = xgb.DMatrix(X_val, y_val)
 d_val.set_base_margin(M[val_ind].flatten())
 del X_train, X_val, y_train, y_val
gc.collect()
model = xgb.train(params, d_train, evals=[(d_train, "train"),(d_val, "eval")], verbose_eval=10, num_boost_round=31)
 y_oof[val_ind] = model.predict(d_val)
 y_test += model.predict(d_test)/NUM_FOLDS
print()
del d_train, d_val
gc.collect()
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=50)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )<find_best_params>
log_loss(df["open_channels"], y_oof )
study.best_trial.params<init_hyperparams>
df["xgb_pred"] = np.argmax(y_oof, axis=1)
 f1_score(df["open_channels"], df["xgb_pred"], average="macro" )
Best_params_xgb = {'lambda': 0.0014311714230223992,
'alpha': 0.008850567457271379,
'colsample_bytree': 0.3,
'subsample': 1.0,
'learning_rate': 0.01,
'max_depth': 20,
'min_child_weight': 245,
'n_estimators': 4000,
'random_state': 48,
'tree_method':'gpu_hist'}<train_model>
f1_score(df.iloc[:5_000_000]["open_channels"], df.iloc[:5_000_000]["xgb_pred"], average="macro" )
train_x, test_x, train_y, test_y = train_test_split(X, y, test_size=0.15,random_state=42)
 model_xgb = xgb.XGBRegressor(**Best_params_xgb)
 model_xgb.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False )<compute_train_metric>
df["ensemble_pred"] = df[["wave_pred", "xgb_pred"]].max(axis=1)
 f1_score(df["open_channels"], df["ensemble_pred"], average="macro" )
preds = model_xgb.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 rmse<prepare_x_and_y>
f1_score(df.iloc[:5_000_000]["open_channels"], df.iloc[:5_000_000]["ensemble_pred"], average="macro" )
test_X = test[features]<predict_on_test>
test_df["xgb_pred"] = np.argmax(y_test, axis=1 )
<save_to_csv><EOS> 
sample_submission = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 sample_submission['open_channels'] = test_df[["wave_pred", "xgb_pred"]].max(axis=1)
 sample_submission.to_csv(f'submission.csv', index=False, float_format='%.4f')
 print(sample_submission.open_channels.mean())
 display(sample_submission.head() )