Kaggle competition: University of Liverpool - Ion Switching
all_data.drop(columns = ['Name', 'Ticket', 'PassengerId', 'Fare'], inplace =True)
 all_data.head()<prepare_x_and_y>
%matplotlib inline
num_col = [col for col in all_data.columns if all_data[col].dtypes != 'object']
num_col.remove('Survived')
 cat_col = [col for col in all_data.columns if all_data[col].dtypes == 'object']
train_final = all_data[:train.shape[0]]
test_final = all_data[train.shape[0]:]
X = train_final.drop(columns = 'Survived')
 y = train_final.Survived<categorify>
data = pd.read_csv('.. /input/data-without-drift/train_clean.csv')
 data.head()
skf = StratifiedKFold(n_splits=5, random_state=2020, shuffle=True)
 numerical_transformer = make_pipeline(StandardScaler())
 categorical_transformer = make_pipeline(SimpleImputer(strategy='most_frequent'),
OneHotEncoder(handle_unknown='ignore',sparse=False))
preprocess = ColumnTransformer(transformers=[ 
('num', numerical_transformer, num_col), 
('cat', categorical_transformer, cat_col)] )<categorify>
df = divide_and_add_features(data['signal'])
 df.head()
preprocess.fit_transform(X,y)
 enc_cat_col = preprocess.named_transformers_['cat']['onehotencoder'].get_feature_names()
labels = np.concatenate([num_col, enc_cat_col])
 X_transformed = pd.DataFrame(preprocess.fit_transform(X), columns=labels)
 X_transformed<define_variables>
x_train, x_test, y_train, y_test = train_test_split(df.values, data['open_channels'].values, test_size=0.2)
 del data, df
print('x_train.shape=', x_train.shape)
 print('x_test.shape=', x_test.shape)
 print('y_train.shape=', y_train.shape)
 print('y_test.shape=', y_test.shape )
random_state = 2020
clf_list = [('LogisticRegression', LogisticRegression(random_state = random_state)) , 
('KNN_Classifier', KNeighborsClassifier()), 
('SVC', SVC(random_state = random_state, probability = True)) , 
('RandomForestClassifier', RandomForestClassifier(random_state = random_state)) , 
('XGBClassifier', XGBClassifier(random_state = random_state)) , 
('AdaBoostClassifier', AdaBoostClassifier(base_estimator = DecisionTreeClassifier(random_state = random_state), random_state = random_state)) ] 
<compute_train_metric>
def create_mpl(shape):

 X_input = L.Input(shape)
 X = L.Dense(150, activation='relu' )(X_input)
 X = L.Dense(150, activation='relu' )(X)
 X = L.Dense(125, activation='relu' )(X)
 X = L.Dense(100, activation='relu' )(X)
 X = L.Dense(75, activation='relu' )(X)
 X = L.Dense(50, activation='relu' )(X)
 X = L.Dense(25, activation='relu' )(X)
 X = L.Dense(11, activation='softmax' )(X)
 model = Model(inputs=X_input, outputs=X)
 return model
mlp = create_mpl(x_train[0].shape)
 mlp.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])
 print(mlp.summary() )
estimators = [('lr', clf_list[0][1]), 
('knn', clf_list[1][1]),('svc', clf_list[2][1]), 
('rf', clf_list[3][1]),('xgb', clf_list[4][1]), 
('adb', clf_list[5][1])]
base_voting_hard = VotingClassifier(estimators = estimators , voting = 'hard')
 base_voting_soft = VotingClassifier(estimators = estimators , voting = 'soft')
 cv_hard = cross_val_score(base_voting_hard, X_transformed, y, cv = skf)
 cv_soft = cross_val_score(base_voting_soft, X_transformed, y, cv = skf)
 print('Baseline Models - Ensemble
--------------------------')
 print('Hard Voting: {}%'.format(np.round(cv_hard.mean() *100, 1)))
 print('Soft Voting: {}%'.format(np.round(cv_soft.mean() *100, 1)) )<train_on_grid>
mlp.fit(x=x_train, y=y_train, epochs=30, batch_size=1024, class_weight=class_weight )
final_pipe = make_pipeline(preprocess, LogisticRegression())
 param_grid = {'logisticregression__max_iter' : [100],
'logisticregression__penalty' : ['l1', 'l2'],
'logisticregression__C' : np.logspace(-2, 2, 20),
'logisticregression__solver' : ['lbfgs', 'liblinear']}
grid_lr = GridSearchCV(final_pipe, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_lr = grid_lr.fit(X, y)
 lr_param = best_grid_lr.best_params_
print(f'best_params_: {lr_param}')
 print(f'score: {best_grid_lr.best_score_:.5f}' )<train_on_grid>
mlp_pred = mlp.predict(x_test)
 print('mlp_pred.shape=', mlp_pred.shape )
lr = LogisticRegression()
param_grid = {'max_iter' : [100],
'penalty' : ['l1', 'l2'],
'C' : np.logspace(-2, 2, 20),
'solver' : ['lbfgs', 'liblinear']}
grid_lr = GridSearchCV(lr, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_lr = grid_lr.fit(X_transformed, y)
 lr_param = best_grid_lr.best_params_
print(f'best_params_: {lr_param}')
 print(f'
score: {best_grid_lr.best_score_:.5f}' )<train_on_grid>
dataset = lgb.Dataset(x_train, label=y_train)
 params = {'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'n_jobs': -1,
'seed': 236,
'num_leaves': 280,
'learning_rate': 0.026623466966581126,
'max_depth': 73,
'lambda_l1': 2.959759088169741,
'lambda_l2': 1.331172832164913,
'bagging_fraction': 0.9655406551472153,
'bagging_freq': 9,
'colsample_bytree': 0.6867118652742716}
knn = KNeighborsClassifier()
final_pipe = make_pipeline(preprocess, knn)
 param_grid = {'kneighborsclassifier__n_neighbors' : np.arange(3, 30, 2),
'kneighborsclassifier__weights': ['uniform', 'distance'],
'kneighborsclassifier__algorithm': ['auto'],
'kneighborsclassifier__p': [1, 2]}
grid_knn = GridSearchCV(final_pipe, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_knn = grid_knn.fit(X, y)
 knn_param = best_grid_knn.best_params_
print(f'best_params_: {knn_param}')
 print(f'score: {best_grid_knn.best_score_:.5f}' )<train_on_grid>
print('Training LGBM...')
 gbc = lgb.train(params, dataset,
num_boost_round=10000,
verbose_eval=100)
 print('LGBM trained!' )
knn = KNeighborsClassifier()
param_grid = {'n_neighbors' : np.arange(3, 30, 2),
'weights': ['uniform', 'distance'],
'algorithm': ['auto'],
'p': [1, 2]}
grid_knn = GridSearchCV(knn, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_knn = grid_knn.fit(X_transformed, y)
 knn_param = best_grid_knn.best_params_
print(f'best_params_: {knn_param}')
 print(f'
score: {best_grid_knn.best_score_:.5f}' )<define_search_space>
gbc_pred = gbc.predict(x_test)
 print('gbc_pred.shape=', gbc_pred.shape )
svc = SVC(probability = True)
 param_grid = tuned_parameters = [{'kernel': ['rbf'],
'gamma': [0.01, 0.1, 0.5, 1, 2, 5],
'C': [.1, 1, 2, 5]},
{'kernel': ['linear'],
'C': [.1, 1, 2, 10]},
{'kernel': ['poly'],
'degree' : [2, 3, 4, 5],
'C': [.1, 1, 10]}]
grid_svc = GridSearchCV(svc, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_svc = grid_svc.fit(X_transformed, y)
 svc_param = best_grid_svc.best_params_
print(f'best_params_: {svc_param}')
 print(f'
score: {best_grid_svc.best_score_:.5f}' )<train_on_grid>
def convert_to_one_hot(pred):

 pred = np.clip(pred, 0, 10)
 pred = 1 - np.abs(pred.reshape(( -1,1)) - np.arange(11))
pred = np.clip(pred, 0, 1)
 return pred
gbc_pred = convert_to_one_hot(gbc_pred)
 print('gbc_pred.shape=', gbc_pred.shape )
rf = RandomForestClassifier(random_state = 2020)
 param_grid = {'n_estimators': [50, 150, 300, 450],
'criterion': ['entropy'],
'bootstrap': [True],
'max_depth': [3, 5, 10],
'max_features': ['auto','sqrt'],
'min_samples_leaf': [2, 3],
'min_samples_split': [2, 3]}
grid_rf = GridSearchCV(rf, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_rf = grid_rf.fit(X_transformed, y)
 rf_param = best_grid_rf.best_params_
print(f'best_params_: {rf_param}')
 print(f'
score: {best_grid_rf.best_score_:.5f}' )<choose_model_class>
f1s = []
alphas = np.linspace(0,1,101)
 for alpha in tqdm(alphas):
y_pred = alpha*mlp_pred +(1 - alpha)*gbc_pred
f1 = f1_score(y_test, np.argmax(y_pred, axis=-1), average='macro')
 f1s.append(f1)
 f1s = np.array(f1s)
 best_alpha = alphas[np.argmax(f1s)]
print('best_f1=', f1s.max())
 print('best_alpha=', best_alpha )
xgb = XGBClassifier(random_state = 2020)
 param_grid = {'n_estimators': [15, 25, 50, 100],
'colsample_bytree': [0.65, 0.75, 0.80],
'max_depth': [None],
'reg_alpha': [1],
'reg_lambda': [1, 2, 5],
'subsample': [0.50, 0.75, 1.00],
'learning_rate': [0.01, 0.1, 0.5],
'gamma': [0.5, 1, 2, 5],
'min_child_weight': [0.01],
'sampling_method': ['uniform']}
grid_xgb = GridSearchCV(xgb, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_xgb = grid_xgb.fit(X_transformed, y)
 xgb_param = best_grid_xgb.best_params_
print(f'best_params_: {xgb_param}')
 print(f'
score: {best_grid_xgb.best_score_:.5f}' )<choose_model_class>
def plot_cm(y_true, y_pred, title):
figsize=(16,16)
 y_pred = y_pred.astype(int)
 cm = confusion_matrix(y_true, y_pred, labels=np.unique(y_true))
cm_sum = np.sum(cm, axis=1, keepdims=True)
 cm_perc = cm / cm_sum.astype(float)* 100
annot = np.empty_like(cm ).astype(str)
 nrows, ncols = cm.shape
for i in range(nrows):
for j in range(ncols):
c = cm[i, j]
p = cm_perc[i, j]
if i == j:
s = cm_sum[i]
annot[i, j] = '%.1f%%
%d/%d' %(p, c, s)
 elif c == 0:
annot[i, j] = ''
else:
annot[i, j] = '%.1f%%
%d' %(p, c)
 cm = pd.DataFrame(cm, index=np.unique(y_true), columns=np.unique(y_true))
cm.index.name = 'Actual'
cm.columns.name = 'Predicted'
fig, ax = plt.subplots(figsize=figsize)
 plt.title(title)
 sns.heatmap(cm, cmap='viridis', annot=annot, fmt='', ax=ax )
adblogis = AdaBoostClassifier(base_estimator = DecisionTreeClassifier(random_state = random_state), random_state=random_state)
 param_grid = {'algorithm': ['SAMME', 'SAMME.R'],
'base_estimator__criterion' : ['gini', 'entropy'],
'base_estimator__splitter' : ['best', 'random'],
'n_estimators': [2, 5, 10, 50],
'learning_rate': [0.01, 0.1, 0.2, 0.3, 1, 2]}
grid_adblogis = GridSearchCV(adblogis, param_grid = param_grid, cv = skf, verbose = False, n_jobs = -1)
 best_grid_adblogis = grid_adblogis.fit(X_transformed, y)
 adblogis_param = best_grid_adblogis.best_params_
print(f'best_params_: {adblogis_param}')
 print(f'
score: {best_grid_adblogis.best_score_:.5f}' )<find_best_params>
def submit_result(mlp, gbc, alpha):
print('Reading data...')
 data = pd.read_csv('.. /input/data-without-drift/test_clean.csv')
 print('Feature engineering...')
 df = divide_and_add_features(data['signal'])
 print('Predicting MLP...')
 mlp_pred = mlp.predict(df.values)
 print('Predicting GBC...')
 gbc_pred = gbc.predict(df.values)
 gbc_pred = convert_to_one_hot(gbc_pred)
 print('Ensembling...')
 y_pred = alpha*mlp_pred +(1 - alpha)*gbc_pred
y_pred = np.argmax(y_pred, axis=-1)
 print('Writing submission...')
 submission = pd.DataFrame()
submission['time'] = data['time']
submission['open_channels'] = y_pred
submission.to_csv('submission.csv', index=False, float_format='%.4f')
 print('Submission finished!' )
<create_dataframe><EOS> 
submit_result(mlp, gbc, best_alpha )