Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<categorify>
BALANCING = False
MODEL_USE = 3
df_train = pd.read_pickle('.. /input/ion-switching-preprocessing/train.pkl')
 df_test = pd.read_pickle('.. /input/ion-switching-preprocessing/test.pkl')
 df_test.loc[df_test['model'] == 1.5, 'model'] = 2
df_train['is_filtered'] = df_train['is_filtered'].astype(np.uint8)
 df_train['model'] = df_train['model'].astype(np.uint8)
 df_test['model'] = df_test['model'].astype(np.uint8)
 for df in [df_train, df_test]:
df.drop(columns=['signal', 'time_scaled', 'signal_processed_kalman', 'batch'], inplace=True)
 df_train['signal_processed_original'] = df_train['signal_processed']
df_test['signal_processed_original'] = df_test['signal_processed']
df_train['signal_processed'] = df_train['signal_processed_denoised']
df_test['signal_processed'] = df_test['signal_processed_denoised']
for df in [df_train, df_test]:
df.drop(columns=['signal_processed_denoised'], inplace=True)
 print(f'Training Set Shape: {df_train.shape}')
 print(f'Test Set Shape: {df_test.shape}')
 print(f'Training Set Memory Usage: {df_train.memory_usage().sum() / 1024 ** 2:.2f} MB')
 print(f'Test Set Memory Usage: {df_test.memory_usage().sum() / 1024 ** 2:.2f} MB' )<find_best_model_class>
def kappa(y_true, y_pred):
return cohen_kappa_score(y_true, y_pred, weights='quadratic' )
class MultiModelGBDTEnsemble:
def __init__(self,lgb_parameters, xgb_parameters, cb_parameters, model):
self.K = 2
self.kf = KFold(n_splits=self.K)
 self.model = model
self.lgb_parameters = lgb_parameters
self.xgb_parameters = xgb_parameters
self.cb_parameters = cb_parameters
def _lightgbm_train_and_predict(self, X_train, y_train, X_test, weight, model):
print(f'
self.lgb_scores = []
self.lgb_oof = np.zeros(X_train.shape[0])
 self.lgb_y_pred = pd.DataFrame(np.zeros(( X_test.shape[0], self.K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, self.K + 1)])
 self.lgb_importance = pd.DataFrame(np.zeros(( X_train.shape[1], self.K)) , columns=[f'Fold_{i}_Importance' for i in range(1, self.K + 1)], index=X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(self.kf.split(X_train, y_train), 1):
print(f'
Fold {fold}')
 lgb_trn_data = lgb.Dataset(X_train.iloc[trn_idx, :], label=y_train.iloc[trn_idx], weight=weight[trn_idx])
 lgb_val_data = lgb.Dataset(X_train.iloc[val_idx, :], label=y_train.iloc[val_idx])
 lgb_model = lgb.train(self.lgb_parameters, lgb_trn_data, valid_sets=[lgb_trn_data, lgb_val_data], verbose_eval=50)
 lgb_oof_predictions = lgb_model.predict(X_train.iloc[val_idx, :], num_iteration=lgb_model.best_iteration)
 self.lgb_oof[val_idx] = lgb_oof_predictions
df_train.loc[X_train.iloc[val_idx, :].index, 'lgb_solo_model_oof'] = lgb_oof_predictions
lgb_test_predictions = lgb_model.predict(X_test, num_iteration=lgb_model.best_iteration)
 self.lgb_y_pred.iloc[:, fold - 1] = lgb_test_predictions
df_test.loc[X_test.index, f'fold{fold}_lgb_solo_model_predictions'] = lgb_test_predictions
self.lgb_importance.iloc[:, fold - 1] = lgb_model.feature_importance(importance_type='gain')
 lgb_score = f1_score(y_train.iloc[val_idx].values, np.round(np.clip(lgb_oof_predictions, y_train.min() , y_train.max())) , average='macro')
 self.lgb_scores.append(lgb_score)
 print('
Solo LGB Fold {} Macro F1-Score {}
'.format(fold, lgb_score))
print('--------------------')
 print(f'Solo LGB Mean Macro F1-Score {np.mean(self.lgb_scores):.6} [STD:{np.std(self.lgb_scores):.6}]')
 print(f'Solo LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_train, np.round(np.clip(self.lgb_oof, y_train.min() , y_train.max())) , average="macro"):.6}')
 print('--------------------')
 def _xgboost_train_and_predict(self, X_train, y_train, X_test, num_boosting_round, early_stopping_rounds, weight, model):
print(f'
self.xgb_scores = []
self.xgb_oof = np.zeros(X_train.shape[0])
 self.xgb_y_pred = pd.DataFrame(np.zeros(( X_test.shape[0], self.K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, self.K + 1)])
 self.xgb_importance = pd.DataFrame(np.zeros(( X_train.shape[1], self.K)) , columns=[f'Fold_{i}_Importance' for i in range(1, self.K + 1)], index=X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(self.kf.split(X_train, y_train), 1):
print(f'
Fold {fold}')
 xgb_trn_data = xgb.DMatrix(X_train.iloc[trn_idx, :], label=y_train.iloc[trn_idx], weight=weight[trn_idx])
 xgb_val_data = xgb.DMatrix(X_train.iloc[val_idx, :], label=y_train.iloc[val_idx])
 xgb_model = xgb.train(self.xgb_parameters, xgb_trn_data, num_boosting_round, evals=[(xgb_trn_data, 'train'),(xgb_val_data, 'val')], verbose_eval=50, early_stopping_rounds=early_stopping_rounds)
 xgb_oof_predictions = xgb_model.predict(xgb.DMatrix(X_train.iloc[val_idx, :]))
self.xgb_oof[val_idx] = xgb_oof_predictions
df_train.loc[X_train.iloc[val_idx, :].index, 'xgb_solo_model_oof'] = xgb_oof_predictions
xgb_test_predictions = xgb_model.predict(xgb.DMatrix(X_test))
self.xgb_y_pred.iloc[:, fold - 1] = xgb_test_predictions
df_test.loc[X_test.index, f'fold{fold}_xgb_solo_model_predictions'] = xgb_test_predictions
self.xgb_importance.iloc[:, fold - 1] = list(xgb_model.get_score(importance_type='gain' ).values())
 xgb_score = f1_score(y_train.iloc[val_idx].values, np.round(np.clip(xgb_oof_predictions, y_train.min() , y_train.max())) , average='macro')
 self.xgb_scores.append(xgb_score)
 print('Solo XGB Fold {} Macro F1-Score {}
'.format(fold, xgb_score))
print('--------------------')
 print(f'Solo XGB Mean Macro F1-Score {np.mean(self.xgb_scores):.6} [STD:{np.std(self.xgb_scores):.6}]')
 print(f'Solo XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_train, np.round(np.clip(self.xgb_oof, y_train.min() , y_train.max())) , average="macro"):.6}')
 print('--------------------')
 def _catboost_train_and_predict(self, X_train, y_train, X_test, weight, model):
print(f'
self.cb_scores = []
self.cb_oof = np.zeros(X_train.shape[0])
 self.cb_y_pred = pd.DataFrame(np.zeros(( X_test.shape[0], self.K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, self.K + 1)])
 self.cb_importance = pd.DataFrame(np.zeros(( X_train.shape[1], self.K)) , columns=[f'Fold_{i}_Importance' for i in range(1, self.K + 1)], index=X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(self.kf.split(X_train, y_train), 1):
print(f'
Fold {fold}')
 cb_trn_data = cb.Pool(X_train.iloc[trn_idx, :], label=y_train.iloc[trn_idx], weight=weight[trn_idx])
 cb_val_data = cb.Pool(X_train.iloc[val_idx, :], label=y_train.iloc[val_idx])
 cb_model = cb.CatBoostRegressor(**self.cb_parameters)
 cb_model.fit(cb_trn_data)
 cb_oof_predictions = cb_model.predict(cb_val_data)
 self.cb_oof[val_idx] = cb_oof_predictions
df_train.loc[X_train.iloc[val_idx, :].index, 'cb_solo_model_oof'] = cb_oof_predictions
cb_test_predictions = cb_model.predict(cb.Pool(X_test))
self.cb_y_pred.iloc[:, fold - 1] = cb_test_predictions
df_test.loc[X_test.index, f'fold{fold}_cb_solo_model_predictions'] = cb_test_predictions
self.cb_importance.iloc[:, fold - 1] = cb_model.get_feature_importance()
cb_score = f1_score(y_train.iloc[val_idx].values, np.round(np.clip(cb_oof_predictions, y_train.min() , y_train.max())) , average='macro')
 self.cb_scores.append(cb_score)
 print('
Solo CB Fold {} Macro F1-Score {}
'.format(fold, cb_score))
print('--------------------')
 print(f'Solo CB Mean Macro F1-Score {np.mean(self.cb_scores):.6} [STD:{np.std(self.cb_scores):.6}]')
 print(f'Solo CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_train, np.round(np.clip(self.cb_oof, y_train.min() , y_train.max())) , average="macro"):.6}')
 print('--------------------')
 def train_and_predict(self,
lgb_X_train, lgb_y_train, lgb_X_test,
xgb_X_train, xgb_y_train, xgb_X_test,
cb_X_train, cb_y_train, cb_X_test,
num_boosting_round=100, early_stopping_rounds=25,
lgb_sample_weight=None, xgb_sample_weight=None, cb_sample_weight=None, model=None):
if self.model == 'LightGBM':
self._lightgbm_train_and_predict(lgb_X_train, lgb_y_train, lgb_X_test, weight=lgb_sample_weight, model=model)
 elif self.model == 'XGBoost':
self._xgboost_train_and_predict(xgb_X_train, xgb_y_train, xgb_X_test, num_boosting_round=num_boosting_round, early_stopping_rounds=early_stopping_rounds, weight=xgb_sample_weight, model=model)
 elif self.model == 'CatBoost':
self._catboost_train_and_predict(cb_X_train, cb_y_train, cb_X_test, weight=cb_sample_weight, model=model)
 elif self.model == 'Blend':
self._lightgbm_train_and_predict(lgb_X_train, lgb_y_train, lgb_X_test, weight=lgb_sample_weight, model=model)
 self._xgboost_train_and_predict(xgb_X_train, xgb_y_train, xgb_X_test, num_boosting_round=num_boosting_round, early_stopping_rounds=early_stopping_rounds, weight=xgb_sample_weight, model=model)
 self._catboost_train_and_predict(cb_X_train, cb_y_train, cb_X_test, weight=cb_sample_weight, model=model)
 self.blend_oof =(self.lgb_oof * 0.34)+(self.xgb_oof * 0.33)+(self.cb_oof * 0.33)
 print(f'Solo Model {model} Equal Blend OOF(Rounded)Macro F1-Score {f1_score(lgb_y_train, np.round(np.clip(self.blend_oof, lgb_y_train.min() , lgb_y_train.max())) , average="macro")}
')
 def plot_importance(self):
if self.model == 'Blend':
self.lgb_importance['Mean_Importance'] = self.lgb_importance.sum(axis=1)/ self.K
self.lgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 self.xgb_importance['Mean_Importance'] = self.xgb_importance.sum(axis=1)/ self.K
self.xgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 self.cb_importance['Mean_Importance'] = self.cb_importance.sum(axis=1)/ self.K
self.cb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 fig, axs = plt.subplots(nrows=3, figsize=(20, 40))
sns.barplot(x='Mean_Importance', y=self.lgb_importance.index, data=self.lgb_importance, ax=axs[0])
 sns.barplot(x='Mean_Importance', y=self.xgb_importance.index, data=self.xgb_importance, ax=axs[1])
 sns.barplot(x='Mean_Importance', y=self.cb_importance.index, data=self.cb_importance, ax=axs[2])
 for i in range(3):
axs[i].set_xlabel('')
 axs[i].tick_params(axis='x', labelsize=20)
 axs[i].tick_params(axis='y', labelsize=20)
 axs[0].set_title(f'Solo LightGBM Feature Importance(Gain)', size=20, pad=20)
 axs[1].set_title(f'Solo XGBoost Feature Importance(Gain)', size=20, pad=20)
 axs[2].set_title(f'Solo CatBoost Feature Importance(Gain)', size=20, pad=20)
 else:
plt.figure(figsize=(20, 6))
if self.model == 'LightGBM':
self.lgb_importance['Mean_Importance'] = self.lgb_importance.sum(axis=1)/ self.K
self.lgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=self.lgb_importance.index, data=self.lgb_importance)
 elif self.model == 'XGBoost':
self.xgb_importance['Mean_Importance'] = self.xgb_importance.sum(axis=1)/ self.K
self.xgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=self.xgb_importance.index, data=self.xgb_importance)
 elif self.model == 'CatBoost':
self.cb_importance['Mean_Importance'] = self.cb_importance.sum(axis=1)/ self.K
self.cb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=self.cb_importance.index, data=self.cb_importance)
 if self.model != 'Blend':
plt.xlabel('')
 plt.tick_params(axis='x', labelsize=20)
 plt.tick_params(axis='y', labelsize=20)
 plt.title(f'Solo {self.model} Feature Importance(Gain)', size=20, pad=20)
 plt.show() 
<prepare_x_and_y>
breeds = pd.read_csv('.. /input/breed_labels.csv')
 colors = pd.read_csv('.. /input/color_labels.csv')
 train = pd.read_csv('.. /input/train/train.csv')
 test = pd.read_csv('.. /input/test/test.csv')
 sub = pd.read_csv('.. /input/test/sample_submission.csv')
 states = pd.read_csv('.. /input/state_labels.csv')
print('
print('------------------------------------------------------')
 model0_lgb_features = ['signal_processed']
model0_lgb_X_train = df_train[df_train['model'] == 0][model0_lgb_features].copy(deep=True)
 model0_lgb_y_train = df_train[df_train['model'] == 0]['open_channels'].copy(deep=True)
 model0_lgb_X_test = df_test[df_test['model'] == 0][model0_lgb_features].copy(deep=True)
 model0_lgb_sample_weight = model0_lgb_y_train.copy(deep=True ).reset_index(drop=True)
 model0_lgb_sample_weight.loc[model0_lgb_sample_weight == 0] = 1
model0_lgb_sample_weight.loc[model0_lgb_sample_weight == 1] = 1
for df in [model0_lgb_X_train, model0_lgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_mean_50'] = df['signal_processed'].rolling(window=50, min_periods=1, center=False ).mean()
df['signal_processed_rolling_forward_center_std_20'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_mean_70_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=70, min_periods=1, center=False ).mean()
df['signal_processed_rolling_forward_center_max_25_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=25, min_periods=1, center=True ).max()
df['signal_processed_diff+1'] = df['signal_processed'].diff(1)
 df['signal_processed_rolling_forward_center_min_35'] = df['signal_processed'].rolling(window=35, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_35'] = df['signal_processed'].rolling(window=35, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_center_maxmin_ratio_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).max() / df['signal_processed'].rolling(window=15, min_periods=1, center=False ).min()
model0_lgb_parameters = {
'num_iterations': 250,
'early_stopping_round': 50,
'num_leaves': 2 ** 7,
'learning_rate': 0.075,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 0.5,
'feature_fraction_bynode': 0.5,
'min_data_in_leaf': 20,
'lambda_l1': 0.5,
'lambda_l2': 0,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model0_xgb_features = ['signal_processed']
model0_xgb_X_train = df_train[df_train['model'] == 0][model0_xgb_features].copy(deep=True)
 model0_xgb_y_train = df_train[df_train['model'] == 0]['open_channels'].copy(deep=True)
 model0_xgb_X_test = df_test[df_test['model'] == 0][model0_xgb_features].copy(deep=True)
 model0_xgb_sample_weight = model0_xgb_y_train.copy(deep=True ).reset_index(drop=True)
 model0_xgb_sample_weight.loc[model0_xgb_sample_weight == 0] = 1
model0_xgb_sample_weight.loc[model0_xgb_sample_weight == 1] = 1
for df in [model0_xgb_X_train, model0_xgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_rolling_forward_mean_50'] = df['signal_processed'].rolling(window=50, min_periods=1, center=False ).mean()
df['signal_processed_rolling_forward_center_std_45'] = df['signal_processed'].rolling(window=45, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_min_45'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_45'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_center_maxminrange_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).max() - df['signal_processed'].rolling(window=30, min_periods=1, center=True ).min()
df['signal_processed_diff+1'] = df['signal_processed'].diff(1 ).fillna(0)
 df['signal_processed_rolling_forward_center_mean_50_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=20, min_periods=1, center=False ).mean()
model0_xgb_parameters = {
'n_estimators': 2 ** 8,
'learning_rate': 0.05,
'colsample_bytree': 0.8,
'colsample_bylevel': 0.8,
'colsample_bynode': 0.8,
'sumbsample': 0.9,
'max_depth': 5,
'gamma': 0.75,
'min_child_weight': 1,
'lambda': 1,
'alpha': 0.1,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model0_cb_features = ['signal_processed']
model0_cb_X_train = df_train[df_train['model'] == 0][model0_cb_features].copy(deep=True)
 model0_cb_y_train = df_train[df_train['model'] == 0]['open_channels'].copy(deep=True)
 model0_cb_X_test = df_test[df_test['model'] == 0][model0_cb_features].copy(deep=True)
 model0_cb_sample_weight = model0_cb_y_train.copy(deep=True ).reset_index(drop=True)
 model0_cb_sample_weight.loc[model0_cb_sample_weight == 0] = 1
model0_cb_sample_weight.loc[model0_cb_sample_weight == 1] = 1
for df in [model0_cb_X_train, model0_cb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_rolling_forward_center_mean_40'] = df['signal_processed'].rolling(window=40, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_min_45'] = df['signal_processed'].rolling(window=45, min_periods=1, center=False ).min()
df['signal_processed_rolling_forward_center_max_50'] = df['signal_processed'].rolling(window=50, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_center_maxminratio_10'] = df['signal_processed'].rolling(window=25, min_periods=1, center=True ).max() / df['signal_processed'].rolling(window=25, min_periods=1, center=True ).min()
model0_cb_parameters = {
'n_estimators': 250,
'learning_rate': 0.06,
'depth': 6,
'subsample': 0.7,
'bagging_temperature': 1,
'colsample_bylevel': 0.6,
'l2_leaf_reg': 0.04,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
solo_model0 = MultiModelGBDTEnsemble(lgb_parameters=model0_lgb_parameters,
xgb_parameters=model0_xgb_parameters,
cb_parameters=model0_cb_parameters,
model='Blend')
 solo_model0.train_and_predict(lgb_X_train=model0_lgb_X_train, lgb_y_train=model0_lgb_y_train, lgb_X_test=model0_lgb_X_test,
xgb_X_train=model0_xgb_X_train, xgb_y_train=model0_xgb_y_train, xgb_X_test=model0_xgb_X_test,
cb_X_train=model0_cb_X_train, cb_y_train=model0_cb_y_train, cb_X_test=model0_cb_X_test,
num_boosting_round=300, early_stopping_rounds=15,
lgb_sample_weight=model0_lgb_sample_weight, xgb_sample_weight=model0_xgb_sample_weight, cb_sample_weight=model0_cb_sample_weight,
model=0)
 solo_model0.plot_importance()
del model0_lgb_X_train, model0_lgb_y_train, model0_lgb_X_test, model0_lgb_sample_weight, model0_lgb_parameters
del model0_xgb_X_train, model0_xgb_y_train, model0_xgb_X_test, model0_xgb_sample_weight, model0_xgb_parameters
del model0_cb_X_train, model0_cb_y_train, model0_cb_X_test, model0_cb_sample_weight, model0_cb_parameters
del solo_model0<prepare_x_and_y>
%matplotlib inline
plt.style.use('ggplot' )
print('
print('------------------------------------------------------')
 model1_lgb_features = ['signal_processed']
model1_lgb_X_train = df_train[df_train['model'] == 1][model1_lgb_features].copy(deep=True)
 model1_lgb_y_train = df_train[df_train['model'] == 1]['open_channels'].copy(deep=True)
 model1_lgb_X_test = df_test[df_test['model'] == 1][model1_lgb_features].copy(deep=True)
 model1_lgb_sample_weight = model1_lgb_y_train.copy(deep=True ).reset_index(drop=True)
 model1_lgb_sample_weight.loc[model1_lgb_sample_weight == 0] = 1
model1_lgb_sample_weight.loc[model1_lgb_sample_weight == 1] = 1
for df in [model1_lgb_X_train, model1_lgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_mean_45'] = df['signal_processed'].rolling(window=45, min_periods=1, center=False ).mean()
df['signal_processed_rolling_forward_std_10'] = df['signal_processed'].rolling(window=20, min_periods=1, center=False ).std()
df['signal_processed_rolling_forward_center_min_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_center_maxmindifference_10'] = df['signal_processed'].rolling(window=5, min_periods=1, center=True ).max() - df['signal_processed'].rolling(window=5, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_min_15_shift-15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).min().shift(-15)
 df['signal_processed_rolling_forward_max_15_shift-15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).max().shift(-15)
 df['signal_processed_rolling_forward_mean_15_shift-15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).mean().shift(-15)
 model1_lgb_parameters = {
'num_iterations': 250,
'early_stopping_round': 50,
'num_leaves': 2 ** 5,
'learning_rate': 0.085,
'bagging_fraction': 0.5,
'bagging_freq': 1,
'feature_fraction': 0.7,
'feature_fraction_bynode': 0.5,
'min_data_in_leaf': 20,
'lambda_l1': 0.1,
'lambda_l2': 0.5,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model1_xgb_features = ['signal_processed']
model1_xgb_X_train = df_train[df_train['model'] == 1][model1_xgb_features].copy(deep=True)
 model1_xgb_y_train = df_train[df_train['model'] == 1]['open_channels'].copy(deep=True)
 model1_xgb_X_test = df_test[df_test['model'] == 1][model1_xgb_features].copy(deep=True)
 model1_xgb_sample_weight = model1_xgb_y_train.copy(deep=True ).reset_index(drop=True)
 model1_xgb_sample_weight.loc[model1_xgb_sample_weight == 0] = 1
model1_xgb_sample_weight.loc[model1_xgb_sample_weight == 1] = 1
for df in [model1_xgb_X_train, model1_xgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_median_25'] = df['signal_processed'].rolling(window=25, min_periods=1, center=False ).median()
df['signal_processed_rolling_forward_std_10'] = df['signal_processed'].rolling(window=25, min_periods=1, center=False ).std()
df['signal_processed_rolling_forward_center_min_10'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=False ).max()
df['signal_processed_rolling_forward_center_maxminratio_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).max() / df['signal_processed'].rolling(window=15, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_min_15_shift-15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).min().shift(-15)
 df['signal_processed_rolling_forward_max_15_shift-15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).max().shift(-15)
 df['signal_processed_rolling_forward_mean_15_shift-15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).mean().shift(-15)
 model1_xgb_parameters = {
'n_estimators': 2 ** 6,
'learning_rate': 0.085,
'colsample_bytree': 0.8,
'colsample_bylevel': 0.8,
'colsample_bynode': 0.7,
'sumbsample': 0.8,
'max_depth': 6,
'gamma': 0.4,
'min_child_weight': 1,
'lambda': 0.6,
'alpha': 0,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model1_cb_features = ['signal_processed']
model1_cb_X_train = df_train[df_train['model'] == 1][model1_cb_features].copy(deep=True)
 model1_cb_y_train = df_train[df_train['model'] == 1]['open_channels'].copy(deep=True)
 model1_cb_X_test = df_test[df_test['model'] == 1][model1_cb_features].copy(deep=True)
 model1_cb_sample_weight = model1_cb_y_train.copy(deep=True ).reset_index(drop=True)
 model1_cb_sample_weight.loc[model1_cb_sample_weight == 0] = 1
model1_cb_sample_weight.loc[model1_cb_sample_weight == 1] = 1
for df in [model1_cb_X_train, model1_cb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_45'] = df['signal_processed'].rolling(window=50, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_std_20'] = df['signal_processed'].rolling(window=20, min_periods=1, center=False ).std()
df['signal_processed_rolling_forward_min_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=False ).min()
df['signal_processed_rolling_forward_center_max_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_min_15_shift-15'] = df['signal_processed'].rolling(window=10 ).min().shift(-10)
 df['signal_processed_rolling_forward_max_15_shift-15'] = df['signal_processed'].rolling(window=10 ).max().shift(-10)
 df['signal_processed_rolling_forward_mean_15_shift-15'] = df['signal_processed'].rolling(window=10 ).mean().shift(-10)
 model1_cb_parameters = {
'n_estimators': 200,
'learning_rate': 0.055,
'depth': 6,
'subsample': 0.6,
'bagging_temperature': 1,
'colsample_bylevel': 0.9,
'l2_leaf_reg': 0.5,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
solo_model1 = MultiModelGBDTEnsemble(lgb_parameters=model1_lgb_parameters,
xgb_parameters=model1_xgb_parameters,
cb_parameters=model1_cb_parameters,
model='Blend')
 solo_model1.train_and_predict(lgb_X_train=model1_lgb_X_train, lgb_y_train=model1_lgb_y_train, lgb_X_test=model1_lgb_X_test,
xgb_X_train=model1_xgb_X_train, xgb_y_train=model1_xgb_y_train, xgb_X_test=model1_xgb_X_test,
cb_X_train=model1_cb_X_train, cb_y_train=model1_cb_y_train, cb_X_test=model1_cb_X_test,
num_boosting_round=250, early_stopping_rounds=10,
lgb_sample_weight=model1_lgb_sample_weight, xgb_sample_weight=model1_xgb_sample_weight, cb_sample_weight=model1_cb_sample_weight,
model=1)
 solo_model1.plot_importance()
del model1_lgb_X_train, model1_lgb_y_train, model1_lgb_X_test, model1_lgb_sample_weight, model1_lgb_parameters
del model1_xgb_X_train, model1_xgb_y_train, model1_xgb_X_test, model1_xgb_sample_weight, model1_xgb_parameters
del model1_cb_X_train, model1_cb_y_train, model1_cb_X_test, model1_cb_sample_weight, model1_cb_parameters
del solo_model1<prepare_x_and_y>
train['AdoptionSpeed'].value_counts()
print('
print('------------------------------------------------------')
 model2_lgb_features = ['signal_processed']
model2_lgb_X_train = df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)][model2_lgb_features].copy(deep=True)
 model2_lgb_y_train = df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model2_lgb_X_test = df_test[df_test['model'] == 2][model2_lgb_features].copy(deep=True)
 model2_lgb_sample_weight = model2_lgb_y_train.copy(deep=True ).reset_index(drop=True)
 model2_lgb_sample_weight.loc[model2_lgb_sample_weight == 0] = 1
model2_lgb_sample_weight.loc[model2_lgb_sample_weight == 1] = 1
model2_lgb_sample_weight.loc[model2_lgb_sample_weight == 2] = 1
model2_lgb_sample_weight.loc[model2_lgb_sample_weight == 3] = 1
for df in [model2_lgb_X_train, model2_lgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_squarerooted'] = df['signal_processed'] **(1 / 2)
 df['signal_processed_rolling_forward_center_mean_45'] = df['signal_processed'].rolling(window=45, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_median_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).median()
df['signal_processed_rolling_forward_center_std_20'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_min_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_35'] = df['signal_processed'].rolling(window=35, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_center_median_50_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=50, min_periods=1, center=True ).median()
df['signal_processed_rolling_forward_min_30_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=30, min_periods=1, center=False ).min()
df['signal_processed_rolling_forward_maxminratio_50'] = df['signal_processed'].rolling(window=50, min_periods=1, center=False ).max() / df['signal_processed'].rolling(window=50, min_periods=1, center=False ).min()
df['signal_processed_diff+1_rolling_forward_center_mean_20'] = df['signal_processed'].diff(1 ).fillna(0 ).rolling(window=20, min_periods=1, center=True ).mean()
model2_lgb_parameters = {
'num_iterations': 250,
'early_stopping_round': 50,
'num_leaves': 2 ** 7,
'learning_rate': 0.05,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 0.9,
'feature_fraction_bynode': 0.5,
'min_data_in_leaf': 20,
'lambda_l1': 0,
'lambda_l2': 0.05,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model2_xgb_features = ['signal_processed']
model2_xgb_X_train = df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)][model2_xgb_features].copy(deep=True)
 model2_xgb_y_train = df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model2_xgb_X_test = df_test[(df_test['model'] == 2)|(df_test['model'] == 1.5)][model2_xgb_features].copy(deep=True)
 model2_xgb_sample_weight = model2_xgb_y_train.copy(deep=True ).reset_index(drop=True)
 model2_xgb_sample_weight.loc[model2_xgb_sample_weight == 0] = 1
model2_xgb_sample_weight.loc[model2_xgb_sample_weight == 1] = 1
model2_xgb_sample_weight.loc[model2_xgb_sample_weight == 2] = 1
model2_xgb_sample_weight.loc[model2_xgb_sample_weight == 3] = 1
for df in [model2_xgb_X_train, model2_xgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_rolling_forward_center_mean_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_std_25'] = df['signal_processed'].rolling(window=25, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_min_40'] = df['signal_processed'].rolling(window=40, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).max()
model2_xgb_parameters = {
'n_estimators': 2 ** 7,
'learning_rate': 0.09,
'colsample_bytree': 0.9,
'colsample_bylevel': 0.9,
'colsample_bynode': 0.8,
'sumbsample': 0.6,
'max_depth': 6,
'gamma': 0.01,
'min_child_weight': 1,
'lambda': 1,
'alpha': 0.005,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model2_cb_features = ['signal_processed']
model2_cb_X_train = df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)][model2_cb_features].copy(deep=True)
 model2_cb_y_train = df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model2_cb_X_test = df_test[(df_test['model'] == 2)|(df_test['model'] == 1.5)][model2_cb_features].copy(deep=True)
 model2_cb_sample_weight = model2_cb_y_train.copy(deep=True ).reset_index(drop=True)
 model2_cb_sample_weight.loc[model2_cb_sample_weight == 0] = 1
model2_cb_sample_weight.loc[model2_cb_sample_weight == 1] = 1
model2_cb_sample_weight.loc[model2_cb_sample_weight == 2] = 1
model2_cb_sample_weight.loc[model2_cb_sample_weight == 3] = 1
for df in [model2_cb_X_train, model2_cb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_rolling_forward_center_mean_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_std_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_min_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_35'] = df['signal_processed'].rolling(window=35, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_mean_30_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=30, min_periods=1, center=False ).mean()
df['signal_processed_rolling_forward_center_min_45_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=45, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_median_45'] = df['signal_processed'].rolling(window=45, min_periods=1, center=True ).median()
df['signal_processed_rolling_forward_min_25_shift-25'] = df['signal_processed'].rolling(window=25, min_periods=1, center=False ).min().shift(-25)
 df['signal_processed_rolling_forward_max_25_shift-25'] = df['signal_processed'].rolling(window=25, min_periods=1, center=False ).max().shift(-25)
 df['signal_processed_rolling_forward_mean_25_shift-25'] = df['signal_processed'].rolling(window=25, min_periods=1, center=False ).mean().shift(-25)
 df['signal_processed_fraction'] = np.abs(df['signal_processed'] - df['signal_processed'].astype(int))
df['signal_processed_squared'] = df['signal_processed'] ** 2
model2_cb_parameters = {
'n_estimators': 300,
'learning_rate': 0.08,
'depth': 6,
'subsample': 0.6,
'bagging_temperature': 1,
'colsample_bylevel': 0.8,
'l2_leaf_reg': 0.06,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
solo_model2 = MultiModelGBDTEnsemble(lgb_parameters=model2_lgb_parameters,
xgb_parameters=model2_xgb_parameters,
cb_parameters=model2_cb_parameters,
model='Blend')
 solo_model2.train_and_predict(lgb_X_train=model2_lgb_X_train, lgb_y_train=model2_lgb_y_train, lgb_X_test=model2_lgb_X_test,
xgb_X_train=model2_xgb_X_train, xgb_y_train=model2_xgb_y_train, xgb_X_test=model2_xgb_X_test,
cb_X_train=model2_cb_X_train, cb_y_train=model2_cb_y_train, cb_X_test=model2_cb_X_test,
num_boosting_round=300, early_stopping_rounds=25,
lgb_sample_weight=model2_lgb_sample_weight, xgb_sample_weight=model2_xgb_sample_weight, cb_sample_weight=model2_cb_sample_weight,
model=2)
 solo_model2.plot_importance()
del model2_lgb_X_train, model2_lgb_y_train, model2_lgb_X_test, model2_lgb_sample_weight, model2_lgb_parameters
del model2_xgb_X_train, model2_xgb_y_train, model2_xgb_X_test, model2_xgb_sample_weight, model2_xgb_parameters
del model2_cb_X_train, model2_cb_y_train, model2_cb_X_test, model2_cb_sample_weight, model2_cb_parameters
del solo_model2<prepare_x_and_y>
states_to_ID = states.set_index('StateName')
 state_value_counts = train['State'].value_counts(ascending=False)
 state_distribution = states_to_ID['StateID'].map(state_value_counts ).sort_values(ascending=False)
 state_distribution
print('
print('------------------------------------------------------')
 model3_lgb_features = ['signal_processed']
model3_lgb_X_train = df_train[df_train['model'] == 3][model3_lgb_features].copy(deep=True)
 model3_lgb_y_train = df_train[df_train['model'] == 3]['open_channels'].copy(deep=True)
 model3_lgb_X_test = df_test[df_test['model'] == 3][model3_lgb_features].copy(deep=True)
 model3_lgb_sample_weight = model3_lgb_y_train.copy(deep=True ).reset_index(drop=True)
 model3_lgb_sample_weight.loc[model3_lgb_sample_weight == 0] = 1
model3_lgb_sample_weight.loc[model3_lgb_sample_weight == 1] = 1
model3_lgb_sample_weight.loc[model3_lgb_sample_weight == 2] = 1
model3_lgb_sample_weight.loc[model3_lgb_sample_weight == 3] = 1
model3_lgb_sample_weight.loc[model3_lgb_sample_weight == 4] = 1
model3_lgb_sample_weight.loc[model3_lgb_sample_weight == 5] = 1
for df in [model3_lgb_X_train, model3_lgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_rolling_forward_mean_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).mean()
df['signal_processed_rolling_forward_std_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).std()
df['signal_processed_rolling_forward_center_min_20'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_35'] = df['signal_processed'].rolling(window=35, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_maxmindifference_35'] = df['signal_processed'].rolling(window=25, min_periods=1, center=False ).max() - df['signal_processed'].rolling(window=25, min_periods=1, center=False ).min()
df['signal_processed_rolling_forward_center_mean_15_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=15, min_periods=1, center=False ).mean()
df['signal_processed_pctchange-1'] = df['signal_processed'].pct_change(-1)
 model3_lgb_parameters = {
'num_iterations': 200,
'early_stopping_round': 50,
'num_leaves': 2 ** 7,
'learning_rate': 0.06,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 0.9,
'feature_fraction_bynode': 0.8,
'min_data_in_leaf': 20,
'lambda_l1': 0.5,
'lambda_l2': 0,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model3_xgb_features = ['signal_processed']
model3_xgb_X_train = df_train[df_train['model'] == 3][model3_xgb_features].copy(deep=True)
 model3_xgb_y_train = df_train[df_train['model'] == 3]['open_channels'].copy(deep=True)
 model3_xgb_X_test = df_test[df_test['model'] == 3][model3_xgb_features].copy(deep=True)
 model3_xgb_sample_weight = model3_xgb_y_train.copy(deep=True ).reset_index(drop=True)
 model3_xgb_sample_weight.loc[model3_xgb_sample_weight == 0] = 1
model3_xgb_sample_weight.loc[model3_xgb_sample_weight == 1] = 1
model3_xgb_sample_weight.loc[model3_xgb_sample_weight == 2] = 1
model3_xgb_sample_weight.loc[model3_xgb_sample_weight == 3] = 1
model3_xgb_sample_weight.loc[model3_xgb_sample_weight == 4] = 1
model3_xgb_sample_weight.loc[model3_xgb_sample_weight == 5] = 1
for df in [model3_xgb_X_train, model3_xgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_rolling_forward_center_mean_35'] = df['signal_processed'].rolling(window=35, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_std_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_min_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_mean_45_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=45, min_periods=1, center=False ).mean()
model3_xgb_parameters = {
'n_estimators': 2 ** 7,
'learning_rate': 0.09,
'colsample_bytree': 0.9,
'colsample_bylevel': 0.9,
'colsample_bynode': 0.9,
'sumbsample': 0.6,
'max_depth': 6,
'gamma': 0,
'min_child_weight': 1,
'lambda': 1,
'alpha': 0,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model3_cb_features = ['signal_processed']
model3_cb_X_train = df_train[df_train['model'] == 3][model3_cb_features].copy(deep=True)
 model3_cb_y_train = df_train[df_train['model'] == 3]['open_channels'].copy(deep=True)
 model3_cb_X_test = df_test[df_test['model'] == 3][model3_cb_features].copy(deep=True)
 model3_cb_sample_weight = model3_cb_y_train.copy(deep=True ).reset_index(drop=True)
 model3_cb_sample_weight.loc[model3_cb_sample_weight == 0] = 1
model3_cb_sample_weight.loc[model3_cb_sample_weight == 1] = 1
model3_cb_sample_weight.loc[model3_cb_sample_weight == 2] = 1
model3_cb_sample_weight.loc[model3_cb_sample_weight == 3] = 1
model3_cb_sample_weight.loc[model3_cb_sample_weight == 4] = 1
model3_cb_sample_weight.loc[model3_cb_sample_weight == 5] = 1
for df in [model3_cb_X_train, model3_cb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_rolling_forward_center_mean_20'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_std_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_min_25'] = df['signal_processed'].rolling(window=25, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_mean_25_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=25, min_periods=1, center=False ).mean()
df['signal_processed_rolling_forward_min_15_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=15, min_periods=1, center=False ).min()
df['signal_processed_rolling_forward_center_maxminratio_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).max() / df['signal_processed'].rolling(window=15, min_periods=1, center=True ).min()
model3_cb_parameters = {
'n_estimators': 550,
'learning_rate': 0.07,
'depth': 6,
'subsample': 0.7,
'bagging_temperature': 1,
'colsample_bylevel': 0.9,
'l2_leaf_reg': 0.05,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
solo_model3 = MultiModelGBDTEnsemble(lgb_parameters=model3_lgb_parameters,
xgb_parameters=model3_xgb_parameters,
cb_parameters=model3_cb_parameters,
model='Blend')
 solo_model3.train_and_predict(lgb_X_train=model3_lgb_X_train, lgb_y_train=model3_lgb_y_train, lgb_X_test=model3_lgb_X_test,
xgb_X_train=model3_xgb_X_train, xgb_y_train=model3_xgb_y_train, xgb_X_test=model3_xgb_X_test,
cb_X_train=model3_cb_X_train, cb_y_train=model3_cb_y_train, cb_X_test=model3_cb_X_test,
num_boosting_round=400, early_stopping_rounds=20,
lgb_sample_weight=model3_lgb_sample_weight, xgb_sample_weight=model3_xgb_sample_weight, cb_sample_weight=model3_cb_sample_weight,
model=3)
 solo_model3.plot_importance()
del model3_lgb_X_train, model3_lgb_y_train, model3_lgb_X_test, model3_lgb_sample_weight, model3_lgb_parameters
del model3_xgb_X_train, model3_xgb_y_train, model3_xgb_X_test, model3_xgb_sample_weight, model3_xgb_parameters
del model3_cb_X_train, model3_cb_y_train, model3_cb_X_test, model3_cb_sample_weight, model3_cb_parameters
del solo_model3<prepare_x_and_y>
train['State'] = train['State'].replace(41401, 41326 )
print('
print('------------------------------------------------------')
 model4_lgb_features = ['signal_processed']
model4_lgb_X_train = df_train[df_train['model'] == 4][model4_lgb_features].copy(deep=True)
 model4_lgb_y_train = df_train[df_train['model'] == 4]['open_channels'].copy(deep=True)
 model4_lgb_X_test = df_test[df_test['model'] == 4][model4_lgb_features].copy(deep=True)
 model4_lgb_sample_weight = model4_lgb_y_train.copy(deep=True ).reset_index(drop=True)
 model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 0] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 1] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 2] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 3] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 4] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 5] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 6] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 7] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 8] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 9] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 10] = 1
for df in [model4_lgb_X_train, model4_lgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_rolling_forward_center_mean_20'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_min_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_center_min_35_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=35, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_min_140'] = df['signal_processed'].rolling(window=140, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_140'] = df['signal_processed'].rolling(window=140, min_periods=1, center=True ).max()
model4_lgb_parameters = {
'num_iterations': 500,
'early_stopping_round': 50,
'num_leaves': 2 ** 7,
'learning_rate': 0.08,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 0.9,
'feature_fraction_bynode': 0.8,
'min_data_in_leaf': 20,
'lambda_l1': 0,
'lambda_l2': 0,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model4_xgb_features = ['signal_processed']
model4_xgb_X_train = df_train[df_train['model'] == 4][model4_xgb_features].copy(deep=True)
 model4_xgb_y_train = df_train[df_train['model'] == 4]['open_channels'].copy(deep=True)
 model4_xgb_X_test = df_test[df_test['model'] == 4][model4_xgb_features].copy(deep=True)
 model4_xgb_sample_weight = model4_xgb_y_train.copy(deep=True ).reset_index(drop=True)
 model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 0] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 1] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 2] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 3] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 4] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 5] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 6] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 7] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 8] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 9] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 10] = 1
for df in [model4_xgb_X_train, model4_xgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_squarerooted'] = df['signal_processed'] **(1 / 2)
 df['signal_processed_rolling_forward_center_mean_20'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_std_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_min_5'] = df['signal_processed'].rolling(window=5, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_mean_45_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=45, min_periods=1, center=True ).mean()
model4_xgb_parameters = {
'n_estimators': 2 ** 7,
'learning_rate': 0.08,
'colsample_bytree': 0.9,
'colsample_bylevel': 0.9,
'colsample_bynode': 0.8,
'sumbsample': 0.9,
'max_depth': 6,
'gamma': 0,
'min_child_weight': 1,
'lambda': 1,
'alpha': 0.5,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model4_cb_features = ['signal_processed']
model4_cb_X_train = df_train[df_train['model'] == 4][model4_cb_features].copy(deep=True)
 model4_cb_y_train = df_train[df_train['model'] == 4]['open_channels'].copy(deep=True)
 model4_cb_X_test = df_test[df_test['model'] == 4][model4_cb_features].copy(deep=True)
 model4_cb_sample_weight = model4_cb_y_train.copy(deep=True ).reset_index(drop=True)
 model4_cb_sample_weight.loc[model4_cb_sample_weight == 0] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 1] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 2] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 3] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 4] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 5] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 6] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 7] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 8] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 9] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 10] = 1
for df in [model4_cb_X_train, model4_cb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_squarerooted'] = df['signal_processed'] **(1 / 2)
 df['signal_processed_rolling_forward_center_mean_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_std_55'] = df['signal_processed'].rolling(window=55, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_max_20'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_center_median_15_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=15, min_periods=1, center=True ).median()
model4_cb_parameters = {
'n_estimators': 450,
'learning_rate': 0.065,
'depth': 6,
'subsample': 0.7,
'bagging_temperature': 1,
'colsample_bylevel': 0.9,
'l2_leaf_reg': 0.01,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
solo_model4 = MultiModelGBDTEnsemble(lgb_parameters=model4_lgb_parameters,
xgb_parameters=model4_xgb_parameters,
cb_parameters=model4_cb_parameters,
model='Blend')
 solo_model4.train_and_predict(lgb_X_train=model4_lgb_X_train, lgb_y_train=model4_lgb_y_train, lgb_X_test=model4_lgb_X_test,
xgb_X_train=model4_xgb_X_train, xgb_y_train=model4_xgb_y_train, xgb_X_test=model4_xgb_X_test,
cb_X_train=model4_cb_X_train, cb_y_train=model4_cb_y_train, cb_X_test=model4_cb_X_test,
lgb_sample_weight=model4_lgb_sample_weight, xgb_sample_weight=model4_xgb_sample_weight, cb_sample_weight=model4_cb_sample_weight,
num_boosting_round=500, early_stopping_rounds=20,
model=4)
 solo_model4.plot_importance()
del model4_lgb_X_train, model4_lgb_y_train, model4_lgb_X_test, model4_lgb_sample_weight, model4_lgb_parameters
del model4_xgb_X_train, model4_xgb_y_train, model4_xgb_X_test, model4_xgb_sample_weight, model4_xgb_parameters
del model4_cb_X_train, model4_cb_y_train, model4_cb_X_test, model4_cb_sample_weight, model4_cb_parameters
del solo_model4<prepare_x_and_y>
train['Description'] = train['Description'].fillna('')
 test['Description'] = test['Description'].fillna('')
 train['desc_len'] = train['Description'].apply(lambda x: len(x))
print('
print('---------------------------------------------------------------')
 for model in range(5):
print(f'
print('----------------------------------')
 y_true = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
lgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'lgb_solo_model_oof']
xgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'xgb_solo_model_oof']
cb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'cb_solo_model_oof']
equal_blend_oof =(lgb_oof * 0.34)+(xgb_oof * 0.33)+(cb_oof * 0.33)
 print(f'Solo LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(lgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Solo XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(xgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Solo CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(cb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Solo Model {model} Equal Blend OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(equal_blend_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'
print('---------------------------------------')
 global_y_true = df_train.loc[df_train['is_filtered'] == 0, 'open_channels']
global_lgb_oof = df_train.loc[df_train['is_filtered'] == 0, 'lgb_solo_model_oof']
global_xgb_oof = df_train.loc[df_train['is_filtered'] == 0, 'xgb_solo_model_oof']
global_cb_oof = df_train.loc[df_train['is_filtered'] == 0, 'cb_solo_model_oof']
global_equal_blend_oof =(global_lgb_oof * 0.34)+(global_xgb_oof * 0.33)+(global_cb_oof * 0.33)
 print(f'Solo LGB Model Global OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_lgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Solo XGB Model Global OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_xgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Solo CB Model Global OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_cb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Solo Model Global Equal Blend OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_equal_blend_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 del global_y_true, global_lgb_oof, global_xgb_oof, global_cb_oof, global_equal_blend_oof<prepare_x_and_y>
target_train = train['AdoptionSpeed']
cleaned_train = train.drop(columns=['Name', 'RescuerID', 'Description', 'PetID', 'AdoptionSpeed'])
 test_pet_ID = test['PetID']
test_X = test.drop(columns=['Name', 'RescuerID', 'Description', 'PetID'])
print('
print('-------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model01_lgb_features = ['signal_processed', 'model']
model01_lgb_X_train = df_train[df_train['model'] <= 1][model01_lgb_features].copy(deep=True)
 model01_lgb_y_train = df_train[df_train['model'] <= 1]['open_channels'].copy(deep=True)
 model01_lgb_X_test = df_test[df_train['model'] <= 1][model01_lgb_features].copy(deep=True)
 for df in [model01_lgb_X_train, model01_lgb_X_test]:
for shift in range(1, 3):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_std_15'] = df.groupby('model')['signal_processed'].rolling(window=15, min_periods=1, center=False ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
model01_lgb_parameters = {
'num_iterations': 500,
'early_stopping_round': 50,
'num_leaves': 2 ** 6,
'learning_rate': 0.03,
'bagging_fraction': 0.6,
'bagging_freq': 1,
'feature_fraction': 0.7,
'feature_fraction_bynode': 0.7,
'min_data_in_leaf': 20,
'lambda_l1': 0.6,
'lambda_l2': 0.6,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model01_lgb_scores = []
model01_lgb_oof = np.zeros(model01_lgb_X_train.shape[0])
 model01_lgb_y_pred = pd.DataFrame(np.zeros(( model01_lgb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model01_lgb_importance = pd.DataFrame(np.zeros(( model01_lgb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model01_lgb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model01_lgb_X_train, model01_lgb_X_train['model']), 1):
print(f'
Fold {fold}')
 model01_lgb_trn_data = lgb.Dataset(model01_lgb_X_train.iloc[trn_idx, :], label=model01_lgb_y_train.iloc[trn_idx], categorical_feature=['model'])
 model01_lgb_val_data = lgb.Dataset(model01_lgb_X_train.iloc[val_idx, :], label=model01_lgb_y_train.iloc[val_idx], categorical_feature=['model'])
 model01_lgb_model = lgb.train(model01_lgb_parameters, model01_lgb_trn_data, valid_sets=[model01_lgb_trn_data, model01_lgb_val_data], verbose_eval=50)
 model01_lgb_oof_predictions = model01_lgb_model.predict(model01_lgb_X_train.iloc[val_idx, :], num_iteration=model01_lgb_model.best_iteration)
 model01_lgb_oof[val_idx] = model01_lgb_oof_predictions
df_train.loc[model01_lgb_X_train.iloc[val_idx, :].index, 'lgb_duo_model_oof'] = model01_lgb_oof_predictions
model01_lgb_test_predictions = model01_lgb_model.predict(model01_lgb_X_test, num_iteration=model01_lgb_model.best_iteration)
 model01_lgb_y_pred.iloc[:, fold - 1] = model01_lgb_test_predictions
df_test.loc[model01_lgb_X_test.index, f'fold{fold}_lgb_duo_model_predictions'] = model01_lgb_test_predictions
model01_lgb_importance.iloc[:, fold - 1] = model01_lgb_model.feature_importance(importance_type='gain')
 model01_lgb_score = f1_score(model01_lgb_y_train.iloc[val_idx].values, np.round(np.clip(model01_lgb_oof_predictions, model01_lgb_y_train.min() , model01_lgb_y_train.max())) , average='macro')
 model01_lgb_scores.append(model01_lgb_score)
 print('
Duo LGB Fold {} Macro F1-Score {}
'.format(fold, model01_lgb_score))
print('--------------------')
 print(f'Duo LGB Mean Macro F1-Score {np.mean(model01_lgb_scores):.6} [STD:{np.std(model01_lgb_scores):.6}]')
 for model in [0, 1]:
model_oof = df_train.loc[df_train['model'] == model, 'lgb_duo_model_oof']
model_labels = df_train.loc[df_train['model'] == model, 'open_channels']
print(f'Duo LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model01_lgb_y_train.min() , model01_lgb_y_train.max())) , average="macro"):.6}')
 print(f'Duo LGB Model 0-1 OOF(Rounded)Macro F1-Score {f1_score(model01_lgb_y_train, np.round(np.clip(model01_lgb_oof, model01_lgb_y_train.min() , model01_lgb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 6))
model01_lgb_importance['Mean_Importance'] = model01_lgb_importance.sum(axis=1)/ K
model01_lgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model01_lgb_importance.index, data=model01_lgb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Duo LightGBM Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model01_lgb_X_train, model01_lgb_y_train, model01_lgb_X_test, model01_lgb_parameters
del model01_lgb_scores, model01_lgb_oof, model01_lgb_y_pred, model01_lgb_importance
del model01_lgb_trn_data, model01_lgb_val_data, model01_lgb_model, model01_lgb_oof_predictions, model01_lgb_test_predictions<prepare_x_and_y>
target_train.isnull().values.any()
print('
print('-------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model01_xgb_features = ['signal_processed', 'model']
model01_xgb_X_train = df_train[df_train['model'] <= 1][model01_xgb_features].copy(deep=True)
 model01_xgb_y_train = df_train[df_train['model'] <= 1]['open_channels'].copy(deep=True)
 model01_xgb_X_test = df_test[df_train['model'] <= 1][model01_xgb_features].copy(deep=True)
 for df in [model01_xgb_X_train, model01_xgb_X_test]:
for shift in range(1, 3):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_std_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=False ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=False ).max().reset_index(level='model')['signal_processed']
df['signal_processed_pct_change+1'] = df.groupby('model')['signal_processed'].pct_change(1 ).fillna(0)
 model01_xgb_parameters = {
'n_estimators': 2 ** 7,
'learning_rate': 0.07,
'colsample_bytree': 0.8,
'colsample_bylevel': 0.8,
'colsample_bynode': 0.7,
'sumbsample': 0.7,
'max_depth': 6,
'gamma': 0.8,
'min_child_weight': 1,
'lambda': 1,
'alpha': 0.1,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model01_xgb_scores = []
model01_xgb_oof = np.zeros(model01_xgb_X_train.shape[0])
 model01_xgb_y_pred = pd.DataFrame(np.zeros(( model01_xgb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model01_xgb_importance = pd.DataFrame(np.zeros(( model01_xgb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model01_xgb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model01_xgb_X_train, model01_xgb_X_train['model']), 1):
print(f'
Fold {fold}')
 model01_xgb_trn_data = xgb.DMatrix(model01_xgb_X_train.iloc[trn_idx, :], label=model01_xgb_y_train.iloc[trn_idx])
 model01_xgb_val_data = xgb.DMatrix(model01_xgb_X_train.iloc[val_idx, :], label=model01_xgb_y_train.iloc[val_idx])
 model01_xgb_model = xgb.train(model01_xgb_parameters, model01_xgb_trn_data, 500, evals=[(model01_xgb_trn_data, 'train'),(model01_xgb_val_data, 'val')], verbose_eval=50, early_stopping_rounds=25)
 model01_xgb_oof_predictions = model01_xgb_model.predict(xgb.DMatrix(model01_xgb_X_train.iloc[val_idx, :]))
model01_xgb_oof[val_idx] = model01_xgb_oof_predictions
df_train.loc[model01_xgb_X_train.iloc[val_idx, :].index, 'xgb_duo_model_oof'] = model01_xgb_oof_predictions
model01_xgb_test_predictions = model01_xgb_model.predict(xgb.DMatrix(model01_xgb_X_test))
model01_xgb_y_pred.iloc[:, fold - 1] = model01_xgb_test_predictions
df_test.loc[model01_xgb_X_test.index, f'fold{fold}_xgb_duo_model_predictions'] = model01_xgb_test_predictions
model01_xgb_importance.iloc[:, fold - 1] = list(model01_xgb_model.get_score(importance_type='gain' ).values())
 model01_xgb_score = f1_score(model01_xgb_y_train.iloc[val_idx].values, np.round(np.clip(model01_xgb_oof_predictions, model01_xgb_y_train.min() , model01_xgb_y_train.max())) , average='macro')
 model01_xgb_scores.append(model01_xgb_score)
 print('
Duo XGB Fold {} Macro F1-Score {}
'.format(fold, model01_xgb_score))
print('--------------------')
 print(f'Duo XGB Mean Macro F1-Score {np.mean(model01_xgb_scores):.6} [STD:{np.std(model01_xgb_scores):.6}]')
 for model in [0, 1]:
model_oof = df_train.loc[df_train['model'] == model, 'xgb_duo_model_oof']
model_labels = df_train.loc[df_train['model'] == model, 'open_channels']
print(f'Duo XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model01_xgb_y_train.min() , model01_xgb_y_train.max())) , average="macro"):.6}')
 print(f'Duo XGB Model 0-1 OOF(Rounded)Macro F1-Score {f1_score(model01_xgb_y_train, np.round(np.clip(model01_xgb_oof, model01_xgb_y_train.min() , model01_xgb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 6))
model01_xgb_importance['Mean_Importance'] = model01_xgb_importance.sum(axis=1)/ K
model01_xgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model01_xgb_importance.index, data=model01_xgb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Duo XGBoost Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model01_xgb_X_train, model01_xgb_y_train, model01_xgb_X_test, model01_xgb_parameters
del model01_xgb_scores, model01_xgb_oof, model01_xgb_y_pred, model01_xgb_importance
del model01_xgb_trn_data, model01_xgb_val_data, model01_xgb_model, model01_xgb_oof_predictions, model01_xgb_test_predictions<prepare_x_and_y>
test_X.isnull().values.any()
print('
print('-------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model01_cb_features = ['signal_processed', 'model']
model01_cb_X_train = df_train[df_train['model'] <= 1][model01_cb_features].copy(deep=True)
 model01_cb_y_train = df_train[df_train['model'] <= 1]['open_channels'].copy(deep=True)
 model01_cb_X_test = df_test[df_train['model'] <= 1][model01_cb_features].copy(deep=True)
 for df in [model01_cb_X_train, model01_cb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_15'] = df.groupby('model')['signal_processed'].rolling(window=15, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_15'] = df.groupby('model')['signal_processed'].rolling(window=60, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_35_difference'] = df['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=False ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_35_difference'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=False ).max().reset_index(level='model')['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=False ).min().reset_index(level='model')['signal_processed']
model01_cb_parameters = {
'n_estimators': 400,
'learning_rate': 0.08,
'depth': 6,
'subsample': 0.9,
'bagging_temperature': 1,
'colsample_bylevel': 0.9,
'l2_leaf_reg': 0,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
model01_cb_scores = []
model01_cb_oof = np.zeros(model01_cb_X_train.shape[0])
 model01_cb_y_pred = pd.DataFrame(np.zeros(( model01_cb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model01_cb_importance = pd.DataFrame(np.zeros(( model01_cb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model01_cb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model01_cb_X_train, model01_cb_X_train['model']), 1):
print(f'
Fold {fold}')
 model01_cb_trn_data = cb.Pool(model01_cb_X_train.iloc[trn_idx, :], label=model01_cb_y_train.iloc[trn_idx], cat_features=[1])
 model01_cb_val_data = cb.Pool(model01_cb_X_train.iloc[val_idx, :], label=model01_cb_y_train.iloc[val_idx], cat_features=[1])
 model01_cb_model = cb.CatBoostRegressor(**model01_cb_parameters)
 model01_cb_model.fit(model01_cb_trn_data)
 model01_cb_oof_predictions = model01_cb_model.predict(model01_cb_val_data)
 model01_cb_oof[val_idx] = model01_cb_oof_predictions
df_train.loc[model01_cb_X_train.iloc[val_idx, :].index, 'cb_duo_model_oof'] = model01_cb_oof_predictions
model01_cb_test_predictions = model01_cb_model.predict(cb.Pool(model01_cb_X_test, cat_features=[1]))
model01_cb_y_pred.iloc[:, fold - 1] = model01_cb_test_predictions
df_test.loc[model01_cb_X_test.index, f'fold{fold}_cb_duo_model_predictions'] = model01_cb_test_predictions
model01_cb_importance.iloc[:, fold - 1] = model01_cb_model.get_feature_importance()
model01_cb_score = f1_score(model01_cb_y_train.iloc[val_idx].values, np.round(np.clip(model01_cb_oof_predictions, model01_cb_y_train.min() , model01_cb_y_train.max())) , average='macro')
 model01_cb_scores.append(model01_cb_score)
 print('
Duo CB Fold {} Macro F1-Score {}
'.format(fold, model01_cb_score))
print('--------------------')
 print(f'Duo CB Mean Macro F1-Score {np.mean(model01_cb_scores):.6} [STD:{np.std(model01_cb_scores):.6}]')
 for model in [0, 1]:
model_oof = df_train.loc[df_train['model'] == model, 'cb_duo_model_oof']
model_labels = df_train.loc[df_train['model'] == model, 'open_channels']
print(f'Duo CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model01_cb_y_train.min() , model01_cb_y_train.max())) , average="macro"):.6}')
 print(f'Duo CB Model 0-1 OOF(Rounded)Macro F1-Score {f1_score(model01_cb_y_train, np.round(np.clip(model01_cb_oof, model01_cb_y_train.min() , model01_cb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 6))
model01_cb_importance['Mean_Importance'] = model01_cb_importance.sum(axis=1)/ K
model01_cb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model01_cb_importance.index, data=model01_cb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Duo CatBoost Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model01_cb_X_train, model01_cb_y_train, model01_cb_X_test, model01_cb_parameters
del model01_cb_scores, model01_cb_oof, model01_cb_y_pred, model01_cb_importance
del model01_cb_trn_data, model01_cb_val_data, model01_cb_model, model01_cb_oof_predictions, model01_cb_test_predictions<prepare_x_and_y>
seed = 42
print('
print('--------------------------------------------------------------')
 for model in range(2):
print(f'
print('---------------------------------')
 y_true = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
lgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'lgb_duo_model_oof']
xgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'xgb_duo_model_oof']
cb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'cb_duo_model_oof']
equal_blend_oof =(lgb_oof * 0.34)+(xgb_oof * 0.33)+(cb_oof * 0.33)
 print(f'Duo LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(lgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Duo XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(xgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Duo CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(cb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Duo Model {model} Equal Blend OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(equal_blend_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'
print('--------------------------------------')
 global_y_true = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 1), 'open_channels']
global_lgb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 1), 'lgb_duo_model_oof']
global_xgb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 1), 'xgb_duo_model_oof']
global_cb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 1), 'cb_duo_model_oof']
global_equal_blend_oof =(global_lgb_oof * 0.34)+(global_xgb_oof * 0.33)+(global_cb_oof * 0.33)
 print(f'Duo LGB Model Global(0-1)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_lgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Duo XGB Model Global(0-1)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_xgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Duo CB Model Global(0-1)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_cb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Duo Model Global(0-1)Equal Blend OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_equal_blend_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 del global_y_true, global_lgb_oof, global_xgb_oof, global_cb_oof, global_equal_blend_oof<prepare_x_and_y>
class EnsembleModel:
def __init__(self,balancing=False):
self.balance_ratio = 5 if balancing else 1
self.rf_model = RandomForestClassifier()
self.lgb_model = lgb.LGBMClassifier()
self.rand_forest_params= {
'bootstrap': [True, False],
'max_depth': [30,50],
'min_samples_leaf': [20, 30],
'min_samples_split': [10,15],
'n_estimators': [250,300],
'random_state' : [seed]
}
self.lgb_params = {'objective' : ['multi:softprob'],
'eta' : [0.01],
'max_depth' : [7,8],
'num_class' : [5],
'num_leaves':[50],
'lambda' : [0.75],
'reg_alpha':[1e-5, 1e-2],
'silent': [1]
}
self.rf_best_param = None
self.lgb_best_param = None
self.columns = None
def set_scorer(self,kappa):
self.kappa = kappa
self.scorer = make_scorer(kappa)
 def set_param(self,rf_param,lgb_param):
self.rf_best_param = rf_param
self.lgb_best_param = lgb_param
def tune_best_param(self,x_train,y_train):
weights_train = [self.balance_ratio if i==0 else 1 for i in y_train.tolist() ]
rf_gridsearch = GridSearchCV(estimator = self.rf_model,
param_grid = self.rand_forest_params,
cv = 5,
n_jobs = -1,
verbose = 1,
scoring=self.scorer)
 rf_gridsearch.fit(x_train, y_train, sample_weight = weights_train)
 print('tuning for rf finished')
 self.rf_model = rf_gridsearch.best_estimator_
self.rf_best_param = rf_gridsearch.best_params_
lgb_gridsearch = GridSearchCV(self.lgb_model, self.lgb_params, n_jobs=-1,
cv=5,
scoring=self.scorer,
verbose=1, refit=True)
 lgb_gridsearch.fit(x_train, y_train, sample_weight = weights_train)
 print('tuning for lgb finished')
 self.lgb_model = lgb_gridsearch.best_estimator_
self.lgb_best_param = lgb_gridsearch.best_params_
print('best param for rf is:')
 print(self.rf_best_param)
 print('best param for lgb is:')
 print(self.lgb_best_param)
 def _avg(self,y_1,y_2):
return np.rint(( y_1 + y_2)/2.0 ).astype(int)
 def re_fit_with_best_param(self,X,y):
if self.rf_best_param == None or self.lgb_best_param == None:
print('use tune_best_param() method to get best param first')
 return
weights_train = [self.balance_ratio if i==0 else 1 for i in y.tolist() ]
self.rf_model = RandomForestClassifier()
self.lgb_model = lgb.LGBMClassifier()
self.rf_model.set_params(**self.rf_best_param)
 self.lgb_model.set_params(**self.lgb_best_param)
 self.rf_model.fit(X,y,sample_weight=weights_train)
 self.lgb_model.fit(X,y,sample_weight=weights_train)
 print('refit finished')
 def validate(self,x_valid, y_valid):
rf_score = self.kappa(self.rf_model.predict(x_valid), y_valid)
 print('{} score: {}'.format('rf', round(rf_score, 4)))
 lgb_score = self.kappa(self.lgb_model.predict(x_valid), y_valid)
 print('{} score: {}'.format('lgb', round(lgb_score, 4)))
 score = kappa(self._avg(self.lgb_model.predict(x_valid), self.rf_model.predict(x_valid)) , y_valid)
 print('{} score on validation set: {}'.format('combiner', round(score, 4)))
 self.columns = x_valid.columns
def predict(self,test_X):
rf_result = self.rf_model.predict(test_X)
 lgb_result = self.lgb_model.predict(test_X)
 final_result = self._avg(rf_result,lgb_result)
 return final_result
def get_feature_importance(self):
rf_feature_importances = pd.DataFrame({'Feature':self.columns.tolist() ,'importance':self.rf_model.feature_importances_.tolist() })
 lgb_feature_importances = pd.DataFrame({'Feature':self.columns.tolist() ,'importance':self.lgb_model.feature_importances_.tolist() })
 overall_feature_importance = pd.merge(rf_feature_importances, lgb_feature_importances, on='Feature', how='outer')
 overall_feature_importance['avg_importance'] =(overall_feature_importance['importance_x'] + overall_feature_importance['importance_y'])/2
overall_feature_importance = overall_feature_importance.sort_values(by=['avg_importance'], ascending=False)
 return overall_feature_importance
print('
print('----------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model012_lgb_features = ['signal_processed', 'model']
model012_lgb_X_train = df_train[(df_train['model'] <= 2)&(df_train['is_filtered'] == 0)][model012_lgb_features].copy(deep=True)
 model012_lgb_y_train = df_train[(df_train['model'] <= 2)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model012_lgb_X_test = df_test[df_test['model'] <= 2][model012_lgb_features].copy(deep=True)
 for df in [model012_lgb_X_train, model012_lgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_20'] = df.groupby('model')['signal_processed'].rolling(window=20, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_40'] = df.groupby('model')['signal_processed'].rolling(window=40, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_40'] = df.groupby('model')['signal_processed'].rolling(window=40, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_median_35_difference'] = df['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).median().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_50_difference'] = df['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=50, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_maxmin_10_difference'] = df.groupby('model')['signal_processed'].rolling(window=10, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=10, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_squarerooted'] = df['signal_processed'] **(1 / 2)
 model012_lgb_parameters = {
'num_iterations': 300,
'early_stopping_round': 50,
'num_leaves': 2 ** 7,
'learning_rate': 0.045,
'bagging_fraction': 0.7,
'bagging_freq': 1,
'feature_fraction': 0.8,
'feature_fraction_bynode': 0.7,
'min_data_in_leaf': 20,
'lambda_l1': 0,
'lambda_l2': 0.5,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model012_lgb_scores = []
model012_lgb_oof = np.zeros(model012_lgb_X_train.shape[0])
 model012_lgb_y_pred = pd.DataFrame(np.zeros(( model012_lgb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model012_lgb_importance = pd.DataFrame(np.zeros(( model012_lgb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model012_lgb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model012_lgb_X_train, model012_lgb_X_train['model']), 1):
print(f'
Fold {fold}')
 model012_lgb_trn_data = lgb.Dataset(model012_lgb_X_train.iloc[trn_idx, :], label=model012_lgb_y_train.iloc[trn_idx], categorical_feature=['model'])
 model012_lgb_val_data = lgb.Dataset(model012_lgb_X_train.iloc[val_idx, :], label=model012_lgb_y_train.iloc[val_idx], categorical_feature=['model'])
 model012_lgb_model = lgb.train(model012_lgb_parameters, model012_lgb_trn_data, valid_sets=[model012_lgb_trn_data, model012_lgb_val_data], verbose_eval=50)
 model012_lgb_oof_predictions = model012_lgb_model.predict(model012_lgb_X_train.iloc[val_idx, :], num_iteration=model012_lgb_model.best_iteration)
 model012_lgb_oof[val_idx] = model012_lgb_oof_predictions
df_train.loc[model012_lgb_X_train.iloc[val_idx, :].index, 'lgb_trio_model_oof'] = model012_lgb_oof_predictions
model012_lgb_test_predictions = model012_lgb_model.predict(model012_lgb_X_test, num_iteration=model012_lgb_model.best_iteration)
 model012_lgb_y_pred.iloc[:, fold - 1] = model012_lgb_test_predictions
df_test.loc[model012_lgb_X_test.index, f'fold{fold}_lgb_trio_model_predictions'] = model012_lgb_test_predictions
model012_lgb_importance.iloc[:, fold - 1] = model012_lgb_model.feature_importance(importance_type='gain')
 model012_lgb_score = f1_score(model012_lgb_y_train.iloc[val_idx].values, np.round(np.clip(model012_lgb_oof_predictions, model012_lgb_y_train.min() , model012_lgb_y_train.max())) , average='macro')
 model012_lgb_scores.append(model012_lgb_score)
 print('
Trio LGB Fold {} Macro F1-Score {}
'.format(fold, model012_lgb_score))
print('--------------------')
 print(f'Trio LGB Mean Macro F1-Score {np.mean(model012_lgb_scores):.6} [STD:{np.std(model012_lgb_scores):.6}]')
 for model in [0, 1, 2]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'lgb_trio_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Trio LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model012_lgb_y_train.min() , model012_lgb_y_train.max())) , average="macro"):.6}')
 print(f'Trio LGB Model 0-1-2 OOF(Rounded)Macro F1-Score {f1_score(model012_lgb_y_train, np.round(np.clip(model012_lgb_oof, model012_lgb_y_train.min() , model012_lgb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 10))
model012_lgb_importance['Mean_Importance'] = model012_lgb_importance.sum(axis=1)/ K
model012_lgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model012_lgb_importance.index, data=model012_lgb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Trio LightGBM Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model012_lgb_X_train, model012_lgb_y_train, model012_lgb_X_test, model012_lgb_parameters
del model012_lgb_scores, model012_lgb_oof, model012_lgb_y_pred, model012_lgb_importance
del model012_lgb_trn_data, model012_lgb_val_data, model012_lgb_model, model012_lgb_oof_predictions, model012_lgb_test_predictions<prepare_x_and_y>
print('
print('----------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model012_xgb_features = ['signal_processed', 'model']
model012_xgb_X_train = df_train[(df_train['model'] <= 2)&(df_train['is_filtered'] == 0)][model012_xgb_features].copy(deep=True)
 model012_xgb_y_train = df_train[(df_train['model'] <= 2)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model012_xgb_X_test = df_test[df_test['model'] <= 2][model012_xgb_features].copy(deep=True)
 for df in [model012_xgb_X_train, model012_xgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_35'] = df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_35'] = df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_55'] = df.groupby('model')['signal_processed'].rolling(window=55, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_maxminratio_15'] = df.groupby('model')['signal_processed'].rolling(window=15, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed'] / df.groupby('model')['signal_processed'].rolling(window=15, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_squared'] = df['signal_processed'] ** 2
model012_xgb_parameters = {
'n_estimators': 2 ** 7,
'learning_rate': 0.095,
'colsample_bytree': 0.9,
'colsample_bylevel': 0.9,
'colsample_bynode': 0.9,
'sumbsample': 0.7,
'max_depth': 7,
'gamma': 0,
'min_child_weight': 1,
'lambda': 1,
'alpha': 0,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model012_xgb_scores = []
model012_xgb_oof = np.zeros(model012_xgb_X_train.shape[0])
 model012_xgb_y_pred = pd.DataFrame(np.zeros(( model012_xgb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model012_xgb_importance = pd.DataFrame(np.zeros(( model012_xgb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model012_xgb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model012_xgb_X_train, model012_xgb_X_train['model']), 1):
print(f'
Fold {fold}')
 model012_xgb_trn_data = xgb.DMatrix(model012_xgb_X_train.iloc[trn_idx, :], label=model012_xgb_y_train.iloc[trn_idx])
 model012_xgb_val_data = xgb.DMatrix(model012_xgb_X_train.iloc[val_idx, :], label=model012_xgb_y_train.iloc[val_idx])
 model012_xgb_model = xgb.train(model012_xgb_parameters, model012_xgb_trn_data, 500, evals=[(model012_xgb_trn_data, 'train'),(model012_xgb_val_data, 'val')], verbose_eval=50, early_stopping_rounds=25)
 model012_xgb_oof_predictions = model012_xgb_model.predict(xgb.DMatrix(model012_xgb_X_train.iloc[val_idx, :]))
model012_xgb_oof[val_idx] = model012_xgb_oof_predictions
df_train.loc[model012_xgb_X_train.iloc[val_idx, :].index, 'xgb_trio_model_oof'] = model012_xgb_oof_predictions
model012_xgb_test_predictions = model012_xgb_model.predict(xgb.DMatrix(model012_xgb_X_test))
model012_xgb_y_pred.iloc[:, fold - 1] = model012_xgb_test_predictions
df_test.loc[model012_xgb_X_test.index, f'fold{fold}_xgb_trio_model_predictions'] = model012_xgb_test_predictions
model012_xgb_importance.iloc[:, fold - 1] = list(model012_xgb_model.get_score(importance_type='gain' ).values())
 model012_xgb_score = f1_score(model012_xgb_y_train.iloc[val_idx].values, np.round(np.clip(model012_xgb_oof_predictions, model012_xgb_y_train.min() , model012_xgb_y_train.max())) , average='macro')
 model012_xgb_scores.append(model012_xgb_score)
 print('
Trio XGB Fold {} Macro F1-Score {}
'.format(fold, model012_xgb_score))
print('--------------------')
 print(f'Trio XGB Mean Macro F1-Score {np.mean(model012_xgb_scores):.6} [STD:{np.std(model012_xgb_scores):.6}]')
 for model in [0, 1, 2]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'xgb_trio_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Trio XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model012_xgb_y_train.min() , model012_xgb_y_train.max())) , average="macro"):.6}')
 print(f'Trio XGB Model 0-1-2 OOF(Rounded)Macro F1-Score {f1_score(model012_xgb_y_train, np.round(np.clip(model012_xgb_oof, model012_xgb_y_train.min() , model012_xgb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 10))
model012_xgb_importance['Mean_Importance'] = model012_xgb_importance.sum(axis=1)/ K
model012_xgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model012_xgb_importance.index, data=model012_xgb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Trio XGBoost Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model012_xgb_X_train, model012_xgb_y_train, model012_xgb_X_test, model012_xgb_parameters
del model012_xgb_scores, model012_xgb_oof, model012_xgb_y_pred, model012_xgb_importance
del model012_xgb_trn_data, model012_xgb_val_data, model012_xgb_model, model012_xgb_oof_predictions, model012_xgb_test_predictions<prepare_x_and_y>
target_train = train['AdoptionSpeed']
cleaned_train = train.drop(columns=['Name', 'RescuerID', 'Description', 'PetID', 'AdoptionSpeed'])
 test_pet_ID = test['PetID']
test_X = test.drop(columns=['Name', 'RescuerID', 'Description', 'PetID'])
print('
print('----------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model012_cb_features = ['signal_processed', 'model']
model012_cb_X_train = df_train[(df_train['model'] <= 2)&(df_train['is_filtered'] == 0)][model012_cb_features].copy(deep=True)
 model012_cb_y_train = df_train[(df_train['model'] <= 2)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model012_cb_X_test = df_test[df_test['model'] <= 2][model012_cb_features].copy(deep=True)
 for df in [model012_cb_X_train, model012_cb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_20'] = df.groupby('model')['signal_processed'].rolling(window=20, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_35'] = df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_10_difference'] = df['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=10, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_50_difference'] = df['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=50, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_maxmindiff_55'] = df.groupby('model')['signal_processed'].rolling(window=55, min_periods=1, center=False ).max().reset_index(level='model')['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=55, min_periods=1, center=False ).min().reset_index(level='model')['signal_processed']
df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_squarerooted'] = df['signal_processed'] **(1 / 2)
 df['signal_processed_pctchange+1'] = df['signal_processed'].pct_change(1)
 model012_cb_parameters = {
'n_estimators': 500,
'learning_rate': 0.085,
'depth': 7,
'subsample': 0.8,
'bagging_temperature': 1,
'colsample_bylevel': 0.9,
'l2_leaf_reg': 0.6,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
model012_cb_scores = []
model012_cb_oof = np.zeros(model012_cb_X_train.shape[0])
 model012_cb_y_pred = pd.DataFrame(np.zeros(( model012_cb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model012_cb_importance = pd.DataFrame(np.zeros(( model012_cb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model012_cb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model012_cb_X_train, model012_cb_X_train['model']), 1):
print(f'
Fold {fold}')
 model012_cb_trn_data = cb.Pool(model012_cb_X_train.iloc[trn_idx, :], label=model012_cb_y_train.iloc[trn_idx], cat_features=[1])
 model012_cb_val_data = cb.Pool(model012_cb_X_train.iloc[val_idx, :], label=model012_cb_y_train.iloc[val_idx], cat_features=[1])
 model012_cb_model = cb.CatBoostRegressor(**model012_cb_parameters)
 model012_cb_model.fit(model012_cb_trn_data)
 model012_cb_oof_predictions = model012_cb_model.predict(model012_cb_val_data)
 model012_cb_oof[val_idx] = model012_cb_oof_predictions
df_train.loc[model012_cb_X_train.iloc[val_idx, :].index, 'cb_trio_model_oof'] = model012_cb_oof_predictions
model012_cb_test_predictions = model012_cb_model.predict(cb.Pool(model012_cb_X_test, cat_features=[1]))
model012_cb_y_pred.iloc[:, fold - 1] = model012_cb_test_predictions
df_test.loc[model012_cb_X_test.index, f'fold{fold}_cb_trio_model_predictions'] = model012_cb_test_predictions
model012_cb_importance.iloc[:, fold - 1] = model012_cb_model.get_feature_importance()
model012_cb_score = f1_score(model012_cb_y_train.iloc[val_idx].values, np.round(np.clip(model012_cb_oof_predictions, model012_cb_y_train.min() , model012_cb_y_train.max())) , average='macro')
 model012_cb_scores.append(model012_cb_score)
 print('
Trio CB Fold {} Macro F1-Score {}
'.format(fold, model012_cb_score))
print('--------------------')
 print(f'Trio CB Mean Macro F1-Score {np.mean(model012_cb_scores):.6} [STD:{np.std(model012_cb_scores):.6}]')
 for model in [0, 1, 2]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'cb_trio_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Trio CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model012_cb_y_train.min() , model012_cb_y_train.max())) , average="macro"):.6}')
 print(f'Trio CB Model 0-1-2 OOF(Rounded)Macro F1-Score {f1_score(model012_cb_y_train, np.round(np.clip(model012_cb_oof, model012_cb_y_train.min() , model012_cb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 10))
model012_cb_importance['Mean_Importance'] = model012_cb_importance.sum(axis=1)/ K
model012_cb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model012_cb_importance.index, data=model012_cb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Trio CatBoost Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model012_cb_X_train, model012_cb_y_train, model012_cb_X_test, model012_cb_parameters
del model012_cb_scores, model012_cb_oof, model012_cb_y_pred, model012_cb_importance
del model012_cb_trn_data, model012_cb_val_data, model012_cb_model, model012_cb_oof_predictions, model012_cb_test_predictions<prepare_x_and_y>
x_train, x_valid, y_train, y_valid = train_test_split(cleaned_train,
target_train,
test_size=0.2,
random_state=seed)
print('
print('---------------------------------------------------------------')
 for model in range(3):
print(f'
print('----------------------------------')
 y_true = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
lgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'lgb_trio_model_oof']
xgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'xgb_trio_model_oof']
cb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'cb_trio_model_oof']
equal_blend_oof =(lgb_oof * 0.34)+(xgb_oof * 0.33)+(cb_oof * 0.33)
 print(f'Trio LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(lgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Trio XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(xgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Trio CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(cb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Trio Model {model} Equal Blend OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(equal_blend_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'
print('---------------------------------------')
 global_y_true = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 2), 'open_channels']
global_lgb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 2), 'lgb_trio_model_oof']
global_xgb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 2), 'xgb_trio_model_oof']
global_cb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 2), 'cb_trio_model_oof']
global_equal_blend_oof =(global_lgb_oof * 0.34)+(global_xgb_oof * 0.33)+(global_cb_oof * 0.33)
 print(f'Trio LGB Model Global(0-1-2)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_lgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Trio XGB Model Global(0-1-2)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_xgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Trio CB Model Global(0-1-2)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_cb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Trio Model Global(0-1-2)Equal Blend OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_equal_blend_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 del global_y_true, global_lgb_oof, global_xgb_oof, global_cb_oof, global_equal_blend_oof<prepare_x_and_y>
if MODEL_USE == 1 or MODEL_USE==0:
first_model = EnsembleModel(balancing=True)
 first_model.set_scorer(kappa)
 first_model.tune_best_param(x_train, y_train)
 first_model.validate(x_valid, y_valid )
print('
print('------------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model0123_lgb_features = ['signal_processed', 'model']
model0123_lgb_X_train = df_train[(df_train['model'] <= 3)&(df_train['is_filtered'] == 0)][model0123_lgb_features].copy(deep=True)
 model0123_lgb_y_train = df_train[(df_train['model'] <= 3)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model0123_lgb_X_test = df_test[df_test['model'] <= 3][model0123_lgb_features].copy(deep=True)
 for df in [model0123_lgb_X_train, model0123_lgb_X_test]:
for shift in range(1, 5):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_rolling_forward_center_mean_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_50'] = df.groupby('model')['signal_processed'].rolling(window=50, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_60'] = df.groupby('model')['signal_processed'].rolling(window=60, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
model0123_lgb_parameters = {
'num_iterations': 400,
'early_stopping_round': 100,
'num_leaves': 2 ** 7,
'learning_rate': 0.1,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 0.9,
'feature_fraction_bynode': 0.9,
'min_data_in_leaf': 20,
'lambda_l1': 0,
'lambda_l2': 0,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model0123_lgb_scores = []
model0123_lgb_oof = np.zeros(model0123_lgb_X_train.shape[0])
 model0123_lgb_y_pred = pd.DataFrame(np.zeros(( model0123_lgb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model0123_lgb_importance = pd.DataFrame(np.zeros(( model0123_lgb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model0123_lgb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model0123_lgb_X_train, model0123_lgb_X_train['model']), 1):
print(f'
Fold {fold}')
 model0123_lgb_trn_data = lgb.Dataset(model0123_lgb_X_train.iloc[trn_idx, :], label=model0123_lgb_y_train.iloc[trn_idx])
 model0123_lgb_val_data = lgb.Dataset(model0123_lgb_X_train.iloc[val_idx, :], label=model0123_lgb_y_train.iloc[val_idx])
 model0123_lgb_model = lgb.train(model0123_lgb_parameters, model0123_lgb_trn_data, valid_sets=[model0123_lgb_trn_data, model0123_lgb_val_data], verbose_eval=50)
 model0123_lgb_oof_predictions = model0123_lgb_model.predict(model0123_lgb_X_train.iloc[val_idx, :], num_iteration=model0123_lgb_model.best_iteration)
 model0123_lgb_oof[val_idx] = model0123_lgb_oof_predictions
df_train.loc[model0123_lgb_X_train.iloc[val_idx, :].index, 'lgb_quad_model_oof'] = model0123_lgb_oof_predictions
model0123_lgb_test_predictions = model0123_lgb_model.predict(model0123_lgb_X_test, num_iteration=model0123_lgb_model.best_iteration)
 model0123_lgb_y_pred.iloc[:, fold - 1] = model0123_lgb_test_predictions
df_test.loc[model0123_lgb_X_test.index, f'fold{fold}_lgb_quad_model_predictions'] = model0123_lgb_test_predictions
model0123_lgb_importance.iloc[:, fold - 1] = model0123_lgb_model.feature_importance(importance_type='gain')
 model0123_lgb_score = f1_score(model0123_lgb_y_train.iloc[val_idx].values, np.round(np.clip(model0123_lgb_oof_predictions, model0123_lgb_y_train.min() , model0123_lgb_y_train.max())) , average='macro')
 model0123_lgb_scores.append(model0123_lgb_score)
 print('
Quad LGB Fold {} Macro F1-Score {}
'.format(fold, model0123_lgb_score))
print('--------------------')
 print(f'Quad LGB Mean Macro F1-Score {np.mean(model0123_lgb_scores):.6} [STD:{np.std(model0123_lgb_scores):.6}]')
 for model in [0, 1, 2, 3]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'lgb_quad_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Quad LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model0123_lgb_y_train.min() , model0123_lgb_y_train.max())) , average="macro"):.6}')
 print(f'Quad LGB Model 0-1-2-3 OOF(Rounded)Macro F1-Score {f1_score(model0123_lgb_y_train, np.round(np.clip(model0123_lgb_oof, model0123_lgb_y_train.min() , model0123_lgb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 10))
model0123_lgb_importance['Mean_Importance'] = model0123_lgb_importance.sum(axis=1)/ K
model0123_lgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model0123_lgb_importance.index, data=model0123_lgb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Quad LightGBM Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model0123_lgb_X_train, model0123_lgb_y_train, model0123_lgb_X_test, model0123_lgb_parameters
del model0123_lgb_scores, model0123_lgb_oof, model0123_lgb_y_pred, model0123_lgb_importance
del model0123_lgb_trn_data, model0123_lgb_val_data, model0123_lgb_model, model0123_lgb_oof_predictions, model0123_lgb_test_predictions<prepare_x_and_y>
filename = os.listdir(".. /input/train_sentiment")[1]
filename = ".. /input/train_sentiment/"+filename
with open(filename, 'r')as f:
sentiment = json.load(f)
 sentiment
print('
print('------------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model0123_xgb_features = ['signal_processed', 'model']
model0123_xgb_X_train = df_train[(df_train['model'] <= 3)&(df_train['is_filtered'] == 0)][model0123_xgb_features].copy(deep=True)
 model0123_xgb_y_train = df_train[(df_train['model'] <= 3)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model0123_xgb_X_test = df_test[df_test['model'] <= 3][model0123_xgb_features].copy(deep=True)
 for df in [model0123_xgb_X_train, model0123_xgb_X_test]:
for shift in range(1, 3):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_20'] = df.groupby('model')['signal_processed'].rolling(window=20, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_40'] = df.groupby('model')['signal_processed'].rolling(window=40, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
model0123_xgb_parameters = {
'n_estimators': 2 ** 8,
'learning_rate': 0.1,
'colsample_bytree': 0.9,
'colsample_bylevel': 0.9,
'colsample_bynode': 0.9,
'sumbsample': 0.8,
'max_depth': 6,
'gamma': 0,
'min_child_weight': 1,
'lambda': 1,
'alpha': 0,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model0123_xgb_scores = []
model0123_xgb_oof = np.zeros(model0123_xgb_X_train.shape[0])
 model0123_xgb_y_pred = pd.DataFrame(np.zeros(( model0123_xgb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model0123_xgb_importance = pd.DataFrame(np.zeros(( model0123_xgb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model0123_xgb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model0123_xgb_X_train, model0123_xgb_X_train['model']), 1):
print(f'
Fold {fold}')
 model0123_xgb_trn_data = xgb.DMatrix(model0123_xgb_X_train.iloc[trn_idx, :], label=model0123_xgb_y_train.iloc[trn_idx])
 model0123_xgb_val_data = xgb.DMatrix(model0123_xgb_X_train.iloc[val_idx, :], label=model0123_xgb_y_train.iloc[val_idx])
 model0123_xgb_model = xgb.train(model0123_xgb_parameters, model0123_xgb_trn_data, 500, evals=[(model0123_xgb_trn_data, 'train'),(model0123_xgb_val_data, 'val')], verbose_eval=50, early_stopping_rounds=25)
 model0123_xgb_oof_predictions = model0123_xgb_model.predict(xgb.DMatrix(model0123_xgb_X_train.iloc[val_idx, :]))
model0123_xgb_oof[val_idx] = model0123_xgb_oof_predictions
df_train.loc[model0123_xgb_X_train.iloc[val_idx, :].index, 'xgb_quad_model_oof'] = model0123_xgb_oof_predictions
model0123_xgb_test_predictions = model0123_xgb_model.predict(xgb.DMatrix(model0123_xgb_X_test))
model0123_xgb_y_pred.iloc[:, fold - 1] = model0123_xgb_test_predictions
df_test.loc[model0123_xgb_X_test.index, f'fold{fold}_xgb_quad_model_predictions'] = model0123_xgb_test_predictions
model0123_xgb_importance.iloc[:, fold - 1] = list(model0123_xgb_model.get_score(importance_type='gain' ).values())
 model0123_xgb_score = f1_score(model0123_xgb_y_train.iloc[val_idx].values, np.round(np.clip(model0123_xgb_oof_predictions, model0123_xgb_y_train.min() , model0123_xgb_y_train.max())) , average='macro')
 model0123_xgb_scores.append(model0123_xgb_score)
 print('
Quad XGB Fold {} Macro F1-Score {}
'.format(fold, model0123_xgb_score))
print('--------------------')
 print(f'Quad XGB Mean Macro F1-Score {np.mean(model0123_xgb_scores):.6} [STD:{np.std(model0123_xgb_scores):.6}]')
 for model in [0, 1, 2, 3]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'xgb_quad_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Quad XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model0123_xgb_y_train.min() , model0123_xgb_y_train.max())) , average="macro"):.6}')
 print(f'Quad XGB Model 0-1-2-3 OOF(Rounded)Macro F1-Score {f1_score(model0123_xgb_y_train, np.round(np.clip(model0123_xgb_oof, model0123_xgb_y_train.min() , model0123_xgb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 10))
model0123_xgb_importance['Mean_Importance'] = model0123_xgb_importance.sum(axis=1)/ K
model0123_xgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model0123_xgb_importance.index, data=model0123_xgb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Quad XGBoost Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model0123_xgb_X_train, model0123_xgb_y_train, model0123_xgb_X_test, model0123_xgb_parameters
del model0123_xgb_scores, model0123_xgb_oof, model0123_xgb_y_pred, model0123_xgb_importance
del model0123_xgb_trn_data, model0123_xgb_val_data, model0123_xgb_model, model0123_xgb_oof_predictions, model0123_xgb_test_predictions<prepare_x_and_y>
def load_desc_sentiment(path):
all_desc_sentiment_files = os.listdir(path)
 count_file = len(all_desc_sentiment_files)
 desc_sentiment_df = pd.DataFrame(columns=['PetID','desc_senti_magnitude','desc_senti_score'])
 current_file_index = 1
for filename in all_desc_sentiment_files:
with open(path+filename, 'r')as f:
sentiment_json = json.load(f)
 petID = filename.split('.')[0]
magnitude = sentiment_json['documentSentiment']['magnitude']
score = sentiment_json['documentSentiment']['score']
desc_sentiment_df = desc_sentiment_df.append({'PetID': petID, 'desc_senti_magnitude':magnitude,'desc_senti_score':score}, \
ignore_index=True)
 if current_file_index % 1000 == 0 or current_file_index == count_file :
print('current progress: %d file of %d loaded' %(current_file_index,count_file))
current_file_index += 1
return desc_sentiment_df
print('
print('------------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model0123_cb_features = ['signal_processed', 'model']
model0123_cb_X_train = df_train[(df_train['model'] <= 3)&(df_train['is_filtered'] == 0)][model0123_cb_features].copy(deep=True)
 model0123_cb_y_train = df_train[(df_train['model'] <= 3)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model0123_cb_X_test = df_test[df_test['model'] <= 3][model0123_cb_features].copy(deep=True)
 for df in [model0123_cb_X_train, model0123_cb_X_test]:
for shift in range(1, 4):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_rolling_forward_center_mean_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_20'] = df.groupby('model')['signal_processed'].rolling(window=20, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_55'] = df.groupby('model')['signal_processed'].rolling(window=55, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
model0123_cb_parameters = {
'n_estimators': 1000,
'learning_rate': 0.09,
'depth': 7,
'subsample': 0.8,
'bagging_temperature': 1,
'colsample_bylevel': 0.9,
'l2_leaf_reg': 0,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
model0123_cb_scores = []
model0123_cb_oof = np.zeros(model0123_cb_X_train.shape[0])
 model0123_cb_y_pred = pd.DataFrame(np.zeros(( model0123_cb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model0123_cb_importance = pd.DataFrame(np.zeros(( model0123_cb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model0123_cb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model0123_cb_X_train, model0123_cb_X_train['model']), 1):
print(f'
Fold {fold}')
 model0123_cb_trn_data = cb.Pool(model0123_cb_X_train.iloc[trn_idx, :], label=model0123_cb_y_train.iloc[trn_idx], cat_features=[1])
 model0123_cb_val_data = cb.Pool(model0123_cb_X_train.iloc[val_idx, :], label=model0123_cb_y_train.iloc[val_idx], cat_features=[1])
 model0123_cb_model = cb.CatBoostRegressor(**model0123_cb_parameters)
 model0123_cb_model.fit(model0123_cb_trn_data)
 model0123_cb_oof_predictions = model0123_cb_model.predict(model0123_cb_val_data)
 model0123_cb_oof[val_idx] = model0123_cb_oof_predictions
df_train.loc[model0123_cb_X_train.iloc[val_idx, :].index, 'cb_quad_model_oof'] = model0123_cb_oof_predictions
model0123_cb_test_predictions = model0123_cb_model.predict(cb.Pool(model0123_cb_X_test, cat_features=[1]))
model0123_cb_y_pred.iloc[:, fold - 1] = model0123_cb_test_predictions
df_test.loc[model0123_cb_X_test.index, f'fold{fold}_cb_quad_model_predictions'] = model0123_cb_test_predictions
model0123_cb_importance.iloc[:, fold - 1] = model0123_cb_model.get_feature_importance()
model0123_cb_score = f1_score(model0123_cb_y_train.iloc[val_idx].values, np.round(np.clip(model0123_cb_oof_predictions, model0123_cb_y_train.min() , model0123_cb_y_train.max())) , average='macro')
 model0123_cb_scores.append(model0123_cb_score)
 print('
Quad CB Fold {} Macro F1-Score {}
'.format(fold, model0123_cb_score))
print('--------------------')
 print(f'Quad CB Mean Macro F1-Score {np.mean(model0123_cb_scores):.6} [STD:{np.std(model0123_cb_scores):.6}]')
 for model in [0, 1, 2, 3]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'cb_quad_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Quad CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model0123_cb_y_train.min() , model0123_cb_y_train.max())) , average="macro"):.6}')
 print(f'Quad CB Model 0-1-2-3 OOF(Rounded)Macro F1-Score {f1_score(model0123_cb_y_train, np.round(np.clip(model0123_cb_oof, model0123_cb_y_train.min() , model0123_cb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 10))
model0123_cb_importance['Mean_Importance'] = model0123_cb_importance.sum(axis=1)/ K
model0123_cb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model0123_cb_importance.index, data=model0123_cb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Quad CatBoost Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model0123_cb_X_train, model0123_cb_y_train, model0123_cb_X_test, model0123_cb_parameters
del model0123_cb_scores, model0123_cb_oof, model0123_cb_y_pred, model0123_cb_importance
del model0123_cb_trn_data, model0123_cb_val_data, model0123_cb_model, model0123_cb_oof_predictions, model0123_cb_test_predictions<prepare_x_and_y>
tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1, 
)
 tfv.fit(train['Description'])
 desc_X_train = tfv.transform(train['Description'])
 desc_X_test = tfv.transform(test['Description'])
 print(desc_X_train.shape)
 print(desc_X_test.shape )
print('
print('---------------------------------------------------------------')
 for model in range(4):
print(f'
print('----------------------------------')
 y_true = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
lgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'lgb_quad_model_oof']
xgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'xgb_quad_model_oof']
cb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'cb_quad_model_oof']
equal_blend_oof =(lgb_oof * 0.34)+(xgb_oof * 0.33)+(cb_oof * 0.33)
 print(f'Quad LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(lgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Quad XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(xgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Quad CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(cb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Quad Model {model} Equal Blend OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(equal_blend_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'
print('---------------------------------------')
 global_y_true = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 3), 'open_channels']
global_lgb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 3), 'lgb_quad_model_oof']
global_xgb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 3), 'xgb_quad_model_oof']
global_cb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 3), 'cb_quad_model_oof']
global_equal_blend_oof =(global_lgb_oof * 0.34)+(global_xgb_oof * 0.33)+(global_cb_oof * 0.33)
 print(f'Quad LGB Model Global(0-1-2-3)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_lgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Quad XGB Model Global(0-1-2-3)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_xgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Quad CB Model Global(0-1-2-3)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_cb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Quad Model Global(0-1-2-3)Equal Blend OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_equal_blend_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 del global_y_true, global_lgb_oof, global_xgb_oof, global_cb_oof, global_equal_blend_oof<prepare_x_and_y>
train_desc_sentiment_df = load_desc_sentiment(".. /input/train_sentiment/")
 test_desc_sentiment_df = load_desc_sentiment(".. /input/test_sentiment/" )
print('
print('---------------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model01234_lgb_features = ['signal_processed', 'model']
model01234_lgb_X_train = df_train[df_train['is_filtered'] == 0][model01234_lgb_features].copy(deep=True)
 model01234_lgb_y_train = df_train[df_train['is_filtered'] == 0]['open_channels'].copy(deep=True)
 model01234_lgb_X_test = df_test[df_train['is_filtered'] == 0][model01234_lgb_features].copy(deep=True)
 for df in [model01234_lgb_X_train, model01234_lgb_X_test]:
for shift in range(1, 6):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 for diff in range(1, 3):
df[f'signal_processed_diff+{diff}'] = df.groupby('model')['signal_processed'].diff(diff)
 df[f'signal_processed_diff-{diff}'] = df.groupby('model')['signal_processed'].diff(-diff)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_squarerooted'] = df['signal_processed'] **(1 / 2)
 df['signal_processed_abs'] = np.abs(df['signal_processed'])
 df['signal_processed_fraction'] = np.abs(df['signal_processed'] - df['signal_processed'].astype(int))
df['signal_processed_rolling_forward_center_mean_50'] = df.groupby('model')['signal_processed'].rolling(window=50, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_35'] = df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_35'] = df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_var_60'] = df.groupby('model')['signal_processed'].rolling(window=60, min_periods=1, center=True ).var().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_mean_40_difference'] = df['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=40, min_periods=1, center=False ).mean().reset_index(level='model')['signal_processed']
model01234_lgb_parameters = {
'num_iterations': 1500,
'early_stopping_round': 150,
'num_leaves': 2 ** 7,
'learning_rate': 0.025,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 0.8,
'feature_fraction_bynode': 0.6,
'min_data_in_leaf': 20,
'lambda_l1': 0,
'lambda_l2': 0,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model01234_lgb_scores = []
model01234_lgb_oof = np.zeros(model01234_lgb_X_train.shape[0])
 model01234_lgb_y_pred = pd.DataFrame(np.zeros(( model01234_lgb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model01234_lgb_importance = pd.DataFrame(np.zeros(( model01234_lgb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model01234_lgb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model01234_lgb_X_train, model01234_lgb_X_train['model']), 1):
print(f'
Fold {fold}')
 model01234_lgb_trn_data = lgb.Dataset(model01234_lgb_X_train.iloc[trn_idx, :], label=model01234_lgb_y_train.iloc[trn_idx], categorical_feature=['model'])
 model01234_lgb_val_data = lgb.Dataset(model01234_lgb_X_train.iloc[val_idx, :], label=model01234_lgb_y_train.iloc[val_idx], categorical_feature=['model'])
 model01234_lgb_model = lgb.train(model01234_lgb_parameters, model01234_lgb_trn_data, valid_sets=[model01234_lgb_trn_data, model01234_lgb_val_data], verbose_eval=50)
 model01234_lgb_oof_predictions = model01234_lgb_model.predict(model01234_lgb_X_train.iloc[val_idx, :], num_iteration=model01234_lgb_model.best_iteration)
 model01234_lgb_oof[val_idx] = model01234_lgb_oof_predictions
df_train.loc[model01234_lgb_X_train.iloc[val_idx, :].index, 'lgb_penta_model_oof'] = model01234_lgb_oof_predictions
model01234_lgb_test_predictions = model01234_lgb_model.predict(model01234_lgb_X_test, num_iteration=model01234_lgb_model.best_iteration)
 model01234_lgb_y_pred.iloc[:, fold - 1] = model01234_lgb_test_predictions
df_test.loc[model01234_lgb_X_test.index, f'fold{fold}_lgb_penta_model_predictions'] = model01234_lgb_test_predictions
model01234_lgb_importance.iloc[:, fold - 1] = model01234_lgb_model.feature_importance(importance_type='gain')
 model01234_lgb_score = f1_score(model01234_lgb_y_train.iloc[val_idx].values, np.round(np.clip(model01234_lgb_oof_predictions, model01234_lgb_y_train.min() , model01234_lgb_y_train.max())) , average='macro')
 model01234_lgb_scores.append(model01234_lgb_score)
 print('
Penta LGB Fold {} Macro F1-Score {}
'.format(fold, model01234_lgb_score))
print('--------------------')
 print(f'Penta LGB Mean Macro F1-Score {np.mean(model01234_lgb_scores):.6} [STD:{np.std(model01234_lgb_scores):.6}]')
 for model in [0, 1, 2, 3, 4]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'lgb_penta_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Penta LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model01234_lgb_y_train.min() , model01234_lgb_y_train.max())) , average="macro"):.6}')
 print(f'Penta LGB Model 0-1-2-3-4 OOF(Rounded)Macro F1-Score {f1_score(model01234_lgb_y_train, np.round(np.clip(model01234_lgb_oof, model01234_lgb_y_train.min() , model01234_lgb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 16))
model01234_lgb_importance['Mean_Importance'] = model01234_lgb_importance.sum(axis=1)/ K
model01234_lgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model01234_lgb_importance.index, data=model01234_lgb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Penta LightGBM Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model01234_lgb_X_train, model01234_lgb_y_train, model01234_lgb_X_test, model01234_lgb_parameters
del model01234_lgb_scores, model01234_lgb_oof, model01234_lgb_y_pred, model01234_lgb_importance
del model01234_lgb_trn_data, model01234_lgb_val_data, model01234_lgb_model, model01234_lgb_oof_predictions, model01234_lgb_test_predictions<prepare_x_and_y>
train_desc_sentiment_df['score_times_mag'] = train_desc_sentiment_df['desc_senti_magnitude'] * train_desc_sentiment_df['desc_senti_score']
test_desc_sentiment_df['score_times_mag'] = test_desc_sentiment_df['desc_senti_magnitude'] * test_desc_sentiment_df['desc_senti_score']
print('
print('---------------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model01234_cb_features = ['signal_processed', 'model']
model01234_cb_X_train = df_train[df_train['is_filtered'] == 0][model01234_cb_features].copy(deep=True)
 model01234_cb_y_train = df_train[df_train['is_filtered'] == 0]['open_channels'].copy(deep=True)
 model01234_cb_X_test = df_test[df_train['is_filtered'] == 0][model01234_cb_features].copy(deep=True)
 for df in [model01234_cb_X_train, model01234_cb_X_test]:
for shift in range(1, 6):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_squarerooted'] = df['signal_processed'] **(1 / 2)
 df['signal_processed_fraction'] = np.abs(df['signal_processed'] - df['signal_processed'].astype(int))
df['signal_processed_rolling_forward_center_mean_50'] = df.groupby('model')['signal_processed'].rolling(window=50, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_35'] = df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_35'] = df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
model01234_cb_parameters = {
'n_estimators': 1000,
'learning_rate': 0.075,
'depth': 7,
'subsample': 0.6,
'bagging_temperature': 1,
'colsample_bylevel': 0.9,
'l2_leaf_reg': 0.1,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
model01234_cb_scores = []
model01234_cb_oof = np.zeros(model01234_cb_X_train.shape[0])
 model01234_cb_y_pred = pd.DataFrame(np.zeros(( model01234_cb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model01234_cb_importance = pd.DataFrame(np.zeros(( model01234_cb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model01234_cb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model01234_cb_X_train, model01234_cb_X_train['model']), 1):
print(f'
Fold {fold}')
 model01234_cb_trn_data = cb.Pool(model01234_cb_X_train.iloc[trn_idx, :], label=model01234_cb_y_train.iloc[trn_idx], cat_features=[1])
 model01234_cb_val_data = cb.Pool(model01234_cb_X_train.iloc[val_idx, :], label=model01234_cb_y_train.iloc[val_idx], cat_features=[1])
 model01234_cb_model = cb.CatBoostRegressor(**model01234_cb_parameters)
 model01234_cb_model.fit(model01234_cb_trn_data)
 model01234_cb_oof_predictions = model01234_cb_model.predict(model01234_cb_val_data)
 model01234_cb_oof[val_idx] = model01234_cb_oof_predictions
df_train.loc[model01234_cb_X_train.iloc[val_idx, :].index, 'cb_penta_model_oof'] = model01234_cb_oof_predictions
model01234_cb_test_predictions = model01234_cb_model.predict(cb.Pool(model01234_cb_X_test, cat_features=[1]))
model01234_cb_y_pred.iloc[:, fold - 1] = model01234_cb_test_predictions
df_test.loc[model01234_cb_X_test.index, f'fold{fold}_cb_penta_model_predictions'] = model01234_cb_test_predictions
model01234_cb_importance.iloc[:, fold - 1] = model01234_cb_model.get_feature_importance()
model01234_cb_score = f1_score(model01234_cb_y_train.iloc[val_idx].values, np.round(np.clip(model01234_cb_oof_predictions, model01234_cb_y_train.min() , model01234_cb_y_train.max())) , average='macro')
 model01234_cb_scores.append(model01234_cb_score)
 print('
Penta CB Fold {} Macro F1-Score {}
'.format(fold, model01234_cb_score))
print('--------------------')
 print(f'Penta CB Mean Macro F1-Score {np.mean(model01234_cb_scores):.6} [STD:{np.std(model01234_cb_scores):.6}]')
 for model in [0, 1, 2, 3, 4]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'cb_penta_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Penta CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model01234_cb_y_train.min() , model01234_cb_y_train.max())) , average="macro"):.6}')
 print(f'Penta CB Model 0-1-2-3-4 OOF(Rounded)Macro F1-Score {f1_score(model01234_cb_y_train, np.round(np.clip(model01234_cb_oof, model01234_cb_y_train.min() , model01234_cb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 16))
model01234_cb_importance['Mean_Importance'] = model01234_cb_importance.sum(axis=1)/ K
model01234_cb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model01234_cb_importance.index, data=model01234_cb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Penta CatBoost Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model01234_cb_X_train, model01234_cb_y_train, model01234_cb_X_test, model01234_cb_parameters
del model01234_cb_scores, model01234_cb_oof, model01234_cb_y_pred, model01234_cb_importance
del model01234_cb_trn_data, model01234_cb_val_data, model01234_cb_model, model01234_cb_oof_predictions, model01234_cb_test_predictions<compute_train_metric>
desc_X_train = pd.DataFrame(desc_X_train, columns=['desc_{}'.format(i)for i in range(svd.n_components)])
 desc_X_test = pd.DataFrame(desc_X_test, columns=['desc_{}'.format(i)for i in range(svd.n_components)])
 train_with_desc = pd.concat([train,desc_X_train],axis=1)
 test_with_desc = pd.concat([test,desc_X_test],axis=1 )
print('---------- Model 0 ----------
')
 model0_labels = df_train[df_train['model'] == 0]['open_channels'].copy(deep=True)
 model0_solo_lgb_oof_score = f1_score(model0_labels, np.round(df_train[df_train['model'] == 0]['lgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model0_solo_xgb_oof_score = f1_score(model0_labels, np.round(df_train[df_train['model'] == 0]['xgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model0_solo_cb_oof_score = f1_score(model0_labels, np.round(df_train[df_train['model'] == 0]['cb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model0_duo_lgb_oof_score = f1_score(model0_labels, np.round(df_train[df_train['model'] == 0]['lgb_duo_model_oof'] ).astype(np.uint8), average='macro')
 model0_duo_xgb_oof_score = f1_score(model0_labels, np.round(df_train[df_train['model'] == 0]['xgb_duo_model_oof'] ).astype(np.uint8), average='macro')
 model0_duo_cb_oof_score = f1_score(model0_labels, np.round(df_train[df_train['model'] == 0]['cb_duo_model_oof'] ).astype(np.uint8), average='macro')
 df_train.loc[df_train['model'] == 0, 'blend_oof'] =(df_train.loc[df_train['model'] == 0, 'lgb_solo_model_oof'] * 0.17)+\ 
(df_train.loc[df_train['model'] == 0, 'xgb_solo_model_oof'] * 0.165)+\ 
(df_train.loc[df_train['model'] == 0, 'cb_solo_model_oof'] * 0.165)+\ 
(df_train.loc[df_train['model'] == 0, 'lgb_duo_model_oof'] * 0.17)+\ 
(df_train.loc[df_train['model'] == 0, 'xgb_duo_model_oof'] * 0.165)+\ 
(df_train.loc[df_train['model'] == 0, 'cb_duo_model_oof'] * 0.165)
 model0_blend_oof_score = f1_score(model0_labels, np.round(df_train[df_train['model'] == 0]['blend_oof'] ).astype(np.uint8), average='macro')
 print(f'Solo LGB Model 0 OOF(Rounded)Macro F1-Score {model0_solo_lgb_oof_score:.6}')
 print(f'Solo XGB OOF Model 0(Rounded)Macro F1-Score {model0_solo_xgb_oof_score:.6}')
 print(f'Solo CB OOF Model 0(Rounded)Macro F1-Score {model0_solo_cb_oof_score:.6}')
 print(f'Duo LGB Model 0 OOF(Rounded)Macro F1-Score {model0_duo_lgb_oof_score:.6}')
 print(f'Duo XGB OOF Model 0(Rounded)Macro F1-Score {model0_duo_xgb_oof_score:.6}')
 print(f'Duo CB OOF Model 0(Rounded)Macro F1-Score {model0_duo_cb_oof_score:.6}')
 print(f'Blend OOF Model 0(Rounded)Macro F1-Score {model0_blend_oof_score:.6}')
 print('
---------- Model 1 ----------
')
 model1_labels = df_train[df_train['model'] == 1]['open_channels'].copy(deep=True)
 model1_solo_lgb_oof_score = f1_score(model1_labels, np.round(df_train[df_train['model'] == 1]['lgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model1_solo_xgb_oof_score = f1_score(model1_labels, np.round(df_train[df_train['model'] == 1]['xgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model1_solo_cb_oof_score = f1_score(model1_labels, np.round(df_train[df_train['model'] == 1]['cb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model1_duo_lgb_oof_score = f1_score(model1_labels, np.round(df_train[df_train['model'] == 1]['lgb_duo_model_oof'] ).astype(np.uint8), average='macro')
 model1_duo_xgb_oof_score = f1_score(model1_labels, np.round(df_train[df_train['model'] == 1]['xgb_duo_model_oof'] ).astype(np.uint8), average='macro')
 model1_duo_cb_oof_score = f1_score(model1_labels, np.round(df_train[df_train['model'] == 1]['cb_duo_model_oof'] ).astype(np.uint8), average='macro')
 df_train.loc[df_train['model'] == 1, 'blend_oof'] =(df_train.loc[df_train['model'] == 1, 'lgb_solo_model_oof'] * 0.17)+\ 
(df_train.loc[df_train['model'] == 1, 'xgb_solo_model_oof'] * 0.165)+\ 
(df_train.loc[df_train['model'] == 1, 'cb_solo_model_oof'] * 0.165)+\ 
(df_train.loc[df_train['model'] == 1, 'lgb_duo_model_oof'] * 0.17)+\ 
(df_train.loc[df_train['model'] == 1, 'xgb_duo_model_oof'] * 0.165)+\ 
(df_train.loc[df_train['model'] == 1, 'cb_duo_model_oof'] * 0.165)
 model1_blend_oof_score = f1_score(model1_labels, np.round(df_train[df_train['model'] == 1]['blend_oof'] ).astype(np.uint8), average='macro')
 print(f'Solo LGB Model 1 OOF(Rounded)Macro F1-Score {model1_solo_lgb_oof_score:.6}')
 print(f'Solo XGB OOF Model 1(Rounded)Macro F1-Score {model1_solo_xgb_oof_score:.6}')
 print(f'Solo CB OOF Model 1(Rounded)Macro F1-Score {model1_solo_cb_oof_score:.6}')
 print(f'Duo LGB Model 1 OOF(Rounded)Macro F1-Score {model1_duo_lgb_oof_score:.6}')
 print(f'Duo XGB OOF Model 1(Rounded)Macro F1-Score {model1_duo_xgb_oof_score:.6}')
 print(f'Duo CB OOF Model 1(Rounded)Macro F1-Score {model1_duo_cb_oof_score:.6}')
 print(f'Blend OOF Model 1(Rounded)Macro F1-Score {model1_blend_oof_score:.6}')
 print('
---------- Model 2 ----------
')
 model2_labels = df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model2_solo_lgb_oof_score = f1_score(model2_labels, np.round(df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['lgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model2_solo_xgb_oof_score = f1_score(model2_labels, np.round(df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['xgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model2_solo_cb_oof_score = f1_score(model2_labels, np.round(df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['cb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model2_trio_lgb_oof_score = f1_score(model2_labels, np.round(df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['lgb_trio_model_oof'] ).astype(np.uint8), average='macro')
 model2_trio_xgb_oof_score = f1_score(model2_labels, np.round(df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['xgb_trio_model_oof'] ).astype(np.uint8), average='macro')
 model2_trio_cb_oof_score = f1_score(model2_labels, np.round(df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['cb_trio_model_oof'] ).astype(np.uint8), average='macro')
 df_train.loc[(df_train['model'] == 2)&(df_train['is_filtered'] == 0), 'blend_oof'] =(df_train.loc[(df_train['model'] == 2)&(df_train['is_filtered'] == 0), 'lgb_solo_model_oof'] * 0.14)+\ 
(df_train.loc[(df_train['model'] == 2)&(df_train['is_filtered'] == 0), 'xgb_solo_model_oof'] * 0.13)+\ 
(df_train.loc[(df_train['model'] == 2)&(df_train['is_filtered'] == 0), 'cb_solo_model_oof'] * 0.13)+\ 
(df_train.loc[(df_train['model'] == 2)&(df_train['is_filtered'] == 0), 'lgb_trio_model_oof'] * 0.2)+\ 
(df_train.loc[(df_train['model'] == 2)&(df_train['is_filtered'] == 0), 'xgb_trio_model_oof'] * 0.2)+\ 
(df_train.loc[(df_train['model'] == 2)&(df_train['is_filtered'] == 0), 'cb_trio_model_oof'] * 0.2)
 model2_blend_oof_score = f1_score(model2_labels, np.round(df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['blend_oof'] ).astype(np.uint8), average='macro')
 print(f'Solo LGB Model 2 OOF(Rounded)Macro F1-Score {model2_solo_lgb_oof_score:.6}')
 print(f'Solo XGB OOF Model 2(Rounded)Macro F1-Score {model2_solo_xgb_oof_score:.6}')
 print(f'Solo CB OOF Model 2(Rounded)Macro F1-Score {model2_solo_cb_oof_score:.6}')
 print(f'Trio LGB Model 2 OOF(Rounded)Macro F1-Score {model2_trio_lgb_oof_score:.6}')
 print(f'Trio XGB OOF Model 2(Rounded)Macro F1-Score {model2_trio_xgb_oof_score:.6}')
 print(f'Trio CB OOF Model 2(Rounded)Macro F1-Score {model2_trio_cb_oof_score:.6}')
 print(f'Blend OOF Model 2(Rounded)Macro F1-Score {model2_blend_oof_score:.6}')
 print('
---------- Model 3 ----------
')
 model3_labels = df_train[df_train['model'] == 3]['open_channels'].copy(deep=True)
 model3_solo_lgb_oof_score = f1_score(model3_labels, np.round(df_train[df_train['model'] == 3]['lgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model3_solo_xgb_oof_score = f1_score(model3_labels, np.round(df_train[df_train['model'] == 3]['xgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model3_solo_cb_oof_score = f1_score(model3_labels, np.round(df_train[df_train['model'] == 3]['cb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model3_quad_lgb_oof_score = f1_score(model3_labels, np.round(df_train[df_train['model'] == 3]['lgb_quad_model_oof'] ).astype(np.uint8), average='macro')
 model3_quad_xgb_oof_score = f1_score(model3_labels, np.round(df_train[df_train['model'] == 3]['xgb_quad_model_oof'] ).astype(np.uint8), average='macro')
 model3_quad_cb_oof_score = f1_score(model3_labels, np.round(df_train[df_train['model'] == 3]['cb_quad_model_oof'] ).astype(np.uint8), average='macro')
 df_train.loc[df_train['model'] == 3, 'blend_oof'] =(df_train.loc[df_train['model'] == 3, 'lgb_solo_model_oof'] * 0.14)+\ 
(df_train.loc[df_train['model'] == 3, 'xgb_solo_model_oof'] * 0.13)+\ 
(df_train.loc[df_train['model'] == 3, 'cb_solo_model_oof'] * 0.13)+\ 
(df_train.loc[df_train['model'] == 3, 'lgb_quad_model_oof'] * 0.2)+\ 
(df_train.loc[df_train['model'] == 3, 'xgb_quad_model_oof'] * 0.2)+\ 
(df_train.loc[df_train['model'] == 3, 'cb_quad_model_oof'] * 0.2)
 model3_blend_oof_score = f1_score(model3_labels, np.round(df_train[df_train['model'] == 3]['blend_oof'] ).astype(np.uint8), average='macro')
 print(f'Solo LGB Model 3 OOF(Rounded)Macro F1-Score {model3_solo_lgb_oof_score:.6}')
 print(f'Solo XGB OOF Model 3(Rounded)Macro F1-Score {model3_solo_xgb_oof_score:.6}')
 print(f'Solo CB OOF Model 3(Rounded)Macro F1-Score {model3_solo_cb_oof_score:.6}')
 print(f'Quad LGB Model 3 OOF(Rounded)Macro F1-Score {model3_quad_lgb_oof_score:.6}')
 print(f'Quad XGB OOF Model 3(Rounded)Macro F1-Score {model3_quad_xgb_oof_score:.6}')
 print(f'Quad CB OOF Model 3(Rounded)Macro F1-Score {model3_quad_cb_oof_score:.6}')
 print(f'Blend OOF Model 3(Rounded)Macro F1-Score {model3_blend_oof_score:.6}')
 print('
---------- Model 4 ----------
')
 model4_labels = df_train[df_train['model'] == 4]['open_channels'].copy(deep=True)
 model4_solo_lgb_oof_score = f1_score(model4_labels, np.round(df_train[df_train['model'] == 4]['lgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model4_solo_xgb_oof_score = f1_score(model4_labels, np.round(df_train[df_train['model'] == 4]['xgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model4_solo_cb_oof_score = f1_score(model4_labels, np.round(df_train[df_train['model'] == 4]['cb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model4_penta_lgb_oof_score = f1_score(model4_labels, np.round(df_train[df_train['model'] == 4]['lgb_penta_model_oof'] ).astype(np.uint8), average='macro')
 model4_penta_cb_oof_score = f1_score(model4_labels, np.round(df_train[df_train['model'] == 4]['cb_penta_model_oof'] ).astype(np.uint8), average='macro')
 df_train.loc[df_train['model'] == 4, 'blend_oof'] =(df_train.loc[df_train['model'] == 4, 'lgb_solo_model_oof'] * 0.034)+\ 
(df_train.loc[df_train['model'] == 4, 'xgb_solo_model_oof'] * 0.033)+\ 
(df_train.loc[df_train['model'] == 4, 'cb_solo_model_oof'] * 0.033)+\ 
(df_train.loc[df_train['model'] == 4, 'lgb_penta_model_oof'] * 0.45)+\ 
(df_train.loc[df_train['model'] == 4, 'cb_penta_model_oof'] * 0.45)
 model4_blend_oof_score = f1_score(model4_labels, np.round(df_train[df_train['model'] == 4]['blend_oof'] ).astype(np.uint8), average='macro')
 print(f'Solo LGB Model 4 OOF(Rounded)Macro F1-Score {model4_solo_lgb_oof_score:.6}')
 print(f'Solo XGB OOF Model 4(Rounded)Macro F1-Score {model4_solo_xgb_oof_score:.6}')
 print(f'Solo CB OOF Model 4(Rounded)Macro F1-Score {model4_solo_cb_oof_score:.6}')
 print(f'Penta LGB Model 4 OOF(Rounded)Macro F1-Score {model4_penta_lgb_oof_score:.6}')
 print(f'Penta CB OOF Model 4(Rounded)Macro F1-Score {model4_penta_cb_oof_score:.6}')
 print(f'Blend OOF Model 4(Rounded)Macro F1-Score {model4_blend_oof_score:.6}')
 print('
---------- Global ----------
')
 global_blend_oof_score = f1_score(df_train[df_train['is_filtered'] == 0]['open_channels'], np.round(df_train[df_train['is_filtered'] == 0]['blend_oof'] ).astype(np.uint8), average='macro')
 print(f'Global Blend OOF(Rounded)Macro F1-Score {global_blend_oof_score:.6}
' )<feature_engineering>
target_train = train_with_desc['AdoptionSpeed']
joint_train = train_with_desc.merge(train_desc_sentiment_df, how='left',left_on=['PetID'],right_on=['PetID'])
 cleaned_train = joint_train.drop(columns=['Name', 'RescuerID', 'Description', 'PetID', 'AdoptionSpeed'])
 cleaned_train.fillna(0.0,inplace=True)
 test_pet_ID = test_with_desc['PetID']
joint_test = test_with_desc.merge(test_desc_sentiment_df, how='left',left_on=['PetID'],right_on=['PetID'])
 test_X = joint_test.drop(columns=['Name', 'RescuerID', 'Description', 'PetID'])
 test_X.fillna(0.0, inplace=True)
 x_train, x_valid, y_train, y_valid = train_test_split(cleaned_train,
target_train,
test_size=0.2,
random_state=seed )
df_test['lgb_solo_model_predictions'] =(df_test['fold1_lgb_solo_model_predictions'] + df_test['fold2_lgb_solo_model_predictions'])/ 2
df_test['xgb_solo_model_predictions'] =(df_test['fold1_xgb_solo_model_predictions'] + df_test['fold2_xgb_solo_model_predictions'])/ 2
df_test['cb_solo_model_predictions'] =(df_test['fold1_cb_solo_model_predictions'] + df_test['fold2_cb_solo_model_predictions'])/ 2
df_test['lgb_duo_model_predictions'] =(df_test['fold1_lgb_duo_model_predictions'] + df_test['fold2_lgb_duo_model_predictions'])/ 2
df_test['xgb_duo_model_predictions'] =(df_test['fold1_xgb_duo_model_predictions'] + df_test['fold2_xgb_duo_model_predictions'])/ 2
df_test['cb_duo_model_predictions'] =(df_test['fold1_cb_duo_model_predictions'] + df_test['fold2_cb_duo_model_predictions'])/ 2
df_test['lgb_trio_model_predictions'] =(df_test['fold1_lgb_trio_model_predictions'] + df_test['fold2_lgb_trio_model_predictions'])/ 2
df_test['xgb_trio_model_predictions'] =(df_test['fold1_xgb_trio_model_predictions'] + df_test['fold2_xgb_trio_model_predictions'])/ 2
df_test['cb_trio_model_predictions'] =(df_test['fold1_cb_trio_model_predictions'] + df_test['fold2_cb_trio_model_predictions'])/ 2
df_test['lgb_quad_model_predictions'] =(df_test['fold1_lgb_quad_model_predictions'] + df_test['fold2_lgb_quad_model_predictions'])/ 2
df_test['xgb_quad_model_predictions'] =(df_test['fold1_xgb_quad_model_predictions'] + df_test['fold2_xgb_quad_model_predictions'])/ 2
df_test['cb_quad_model_predictions'] =(df_test['fold1_cb_quad_model_predictions'] + df_test['fold2_cb_quad_model_predictions'])/ 2
df_test['lgb_penta_model_predictions'] =(df_test['fold1_lgb_penta_model_predictions'] + df_test['fold2_lgb_penta_model_predictions'])/ 2
df_test['cb_penta_model_predictions'] =(df_test['fold1_cb_penta_model_predictions'] + df_test['fold2_cb_penta_model_predictions'])/ 2
for fold in range(1, 3):
for gbdt in ['lgb', 'xgb', 'cb']:
for model_type in ['solo', 'duo', 'trio', 'quad']:
df_test.drop(columns=[f'fold{fold}_{gbdt}_{model_type}_model_predictions'], inplace=True)
 df_test.drop(columns=['fold1_lgb_penta_model_predictions', 'fold2_lgb_penta_model_predictions', 'fold1_cb_penta_model_predictions', 'fold2_cb_penta_model_predictions'], inplace=True)
 df_test.loc[df_test['model'] == 0, 'blend_predictions'] =(df_test.loc[df_test['model'] == 0, 'lgb_solo_model_predictions'] * 0.17)+\ 
(df_test.loc[df_test['model'] == 0, 'xgb_solo_model_predictions'] * 0.165)+\ 
(df_test.loc[df_test['model'] == 0, 'cb_solo_model_predictions'] * 0.165)+\ 
(df_test.loc[df_test['model'] == 0, 'lgb_duo_model_predictions'] * 0.17)+\ 
(df_test.loc[df_test['model'] == 0, 'xgb_duo_model_predictions'] * 0.165)+\ 
(df_test.loc[df_test['model'] == 0, 'cb_duo_model_predictions'] * 0.165)
 df_test.loc[df_test['model'] == 1, 'blend_predictions'] =(df_test.loc[df_test['model'] == 1, 'lgb_solo_model_predictions'] * 0.17)+\ 
(df_test.loc[df_test['model'] == 1, 'xgb_solo_model_predictions'] * 0.165)+\ 
(df_test.loc[df_test['model'] == 1, 'cb_solo_model_predictions'] * 0.165)+\ 
(df_test.loc[df_test['model'] == 1, 'lgb_duo_model_predictions'] * 0.17)+\ 
(df_test.loc[df_test['model'] == 1, 'xgb_duo_model_predictions'] * 0.165)+\ 
(df_test.loc[df_test['model'] == 1, 'cb_duo_model_predictions'] * 0.165)
 df_test.loc[(df_test['model'] == 1.5)|(df_test['model'] == 2), 'blend_predictions'] =(df_test.loc[(df_test['model'] == 1.5)|(df_test['model'] == 2), 'lgb_solo_model_predictions'] * 0.14)+\ 
(df_test.loc[(df_test['model'] == 1.5)|(df_test['model'] == 2), 'xgb_solo_model_predictions'] * 0.13)+\ 
(df_test.loc[(df_test['model'] == 1.5)|(df_test['model'] == 2), 'cb_solo_model_predictions'] * 0.13)+\ 
(df_test.loc[(df_test['model'] == 1.5)|(df_test['model'] == 2), 'lgb_trio_model_predictions'] * 0.2)+\ 
(df_test.loc[(df_test['model'] == 1.5)|(df_test['model'] == 2), 'xgb_trio_model_predictions'] * 0.2)+\ 
(df_test.loc[(df_test['model'] == 1.5)|(df_test['model'] == 2), 'cb_trio_model_predictions'] * 0.2)
 df_test.loc[df_test['model'] == 3, 'blend_predictions'] =(df_test.loc[df_test['model'] == 3, 'lgb_solo_model_predictions'] * 0.14)+\ 
(df_test.loc[df_test['model'] == 3, 'xgb_solo_model_predictions'] * 0.13)+\ 
(df_test.loc[df_test['model'] == 3, 'cb_solo_model_predictions'] * 0.13)+\ 
(df_test.loc[df_test['model'] == 3, 'lgb_quad_model_predictions'] * 0.2)+\ 
(df_test.loc[df_test['model'] == 3, 'xgb_quad_model_predictions'] * 0.2)+\ 
(df_test.loc[df_test['model'] == 3, 'cb_quad_model_predictions'] * 0.2)
 df_test.loc[df_test['model'] == 4, 'blend_predictions'] =(df_test.loc[df_test['model'] == 4, 'lgb_solo_model_predictions'] * 0.034)+\ 
(df_test.loc[df_test['model'] == 4, 'xgb_solo_model_predictions'] * 0.033)+\ 
(df_test.loc[df_test['model'] == 4, 'cb_solo_model_predictions'] * 0.033)+\ 
(df_test.loc[df_test['model'] == 4, 'lgb_penta_model_predictions'] * 0.45)+\ 
(df_test.loc[df_test['model'] == 4, 'cb_penta_model_predictions'] * 0.45)
 df_test.loc[df_test['model'] == 0, 'blend_predictions'] = np.clip(df_test.loc[df_test['model'] == 0, 'blend_predictions'], 0, 1)
 df_test.loc[df_test['model'] == 1, 'blend_predictions'] = np.clip(df_test.loc[df_test['model'] == 1, 'blend_predictions'], 0, 1)
 df_test.loc[df_test['model'] == 1.5, 'blend_predictions'] = np.clip(df_test.loc[df_test['model'] == 1.5, 'blend_predictions'], 0, 3)
 df_test.loc[df_test['model'] == 2, 'blend_predictions'] = np.clip(df_test.loc[df_test['model'] == 2, 'blend_predictions'], 0, 3)
 df_test.loc[df_test['model'] == 3, 'blend_predictions'] = np.clip(df_test.loc[df_test['model'] == 3, 'blend_predictions'], 0, 5)
 df_test.loc[df_test['model'] == 4, 'blend_predictions'] = np.clip(df_test.loc[df_test['model'] == 4, 'blend_predictions'], 0, 10)
 df_test['blend_predictions'] = df_test['blend_predictions'].round().astype(np.uint8 )<feature_engineering>
if MODEL_USE == 2 or MODEL_USE==0:
second_model = EnsembleModel(balancing=True)
 second_model.set_scorer(kappa)
 second_model.tune_best_param(x_train, y_train)
 second_model.validate(x_valid,y_valid )
df_train.rename(columns={
'lgb_solo_model_oof': 'lgb_solo_model',
'xgb_solo_model_oof': 'xgb_solo_model',
'cb_solo_model_oof': 'cb_solo_model'
}, inplace=True)
 df_train['lgb_multi_model'] = 0
df_train.loc[df_train['model'] == 0, 'lgb_multi_model'] = df_train.loc[df_train['model'] == 0, 'lgb_duo_model_oof']
df_train.loc[df_train['model'] == 1, 'lgb_multi_model'] = df_train.loc[df_train['model'] == 1, 'lgb_duo_model_oof']
df_train.loc[df_train['model'] == 2, 'lgb_multi_model'] = df_train.loc[df_train['model'] == 2, 'lgb_trio_model_oof']
df_train.loc[df_train['model'] == 3, 'lgb_multi_model'] = df_train.loc[df_train['model'] == 3, 'lgb_quad_model_oof']
df_train.loc[df_train['model'] == 4, 'lgb_multi_model'] = df_train.loc[df_train['model'] == 4, 'lgb_penta_model_oof']
df_train['lgb_multi_model'] = df_train['lgb_multi_model'].astype(np.float32)
 df_train['xgb_multi_model'] = 0
df_train.loc[df_train['model'] == 0, 'xgb_multi_model'] = df_train.loc[df_train['model'] == 0, 'xgb_duo_model_oof']
df_train.loc[df_train['model'] == 1, 'xgb_multi_model'] = df_train.loc[df_train['model'] == 1, 'xgb_duo_model_oof']
df_train.loc[df_train['model'] == 2, 'xgb_multi_model'] = df_train.loc[df_train['model'] == 2, 'xgb_trio_model_oof']
df_train.loc[df_train['model'] == 3, 'xgb_multi_model'] = df_train.loc[df_train['model'] == 3, 'xgb_quad_model_oof']
df_train.loc[df_train['model'] == 4, 'xgb_multi_model'] = np.nan
df_train['xgb_multi_model'] = df_train['xgb_multi_model'].astype(np.float32)
 df_train['cb_multi_model'] = 0
df_train.loc[df_train['model'] == 0, 'cb_multi_model'] = df_train.loc[df_train['model'] == 0, 'cb_duo_model_oof']
df_train.loc[df_train['model'] == 1, 'cb_multi_model'] = df_train.loc[df_train['model'] == 1, 'cb_duo_model_oof']
df_train.loc[df_train['model'] == 2, 'cb_multi_model'] = df_train.loc[df_train['model'] == 2, 'cb_trio_model_oof']
df_train.loc[df_train['model'] == 3, 'cb_multi_model'] = df_train.loc[df_train['model'] == 3, 'cb_quad_model_oof']
df_train.loc[df_train['model'] == 4, 'cb_multi_model'] = df_train.loc[df_train['model'] == 4, 'cb_penta_model_oof']
df_train['cb_multi_model'] = df_train['cb_multi_model'].astype(np.float32)
 df_test.rename(columns={
'lgb_solo_model_predictions': 'lgb_solo_model',
'xgb_solo_model_predictions': 'xgb_solo_model',
'cb_solo_model_predictions': 'cb_solo_model'
}, inplace=True)
 df_test['lgb_multi_model'] = 0
df_test.loc[df_test['model'] == 0, 'lgb_multi_model'] = df_test.loc[df_test['model'] == 0, 'lgb_duo_model_predictions']
df_test.loc[df_test['model'] == 1, 'lgb_multi_model'] = df_test.loc[df_test['model'] == 1, 'lgb_duo_model_predictions']
df_test.loc[df_test['model'] == 2, 'lgb_multi_model'] = df_test.loc[df_test['model'] == 2, 'lgb_trio_model_predictions']
df_test.loc[df_test['model'] == 3, 'lgb_multi_model'] = df_test.loc[df_test['model'] == 3, 'lgb_quad_model_predictions']
df_test.loc[df_test['model'] == 4, 'lgb_multi_model'] = df_test.loc[df_test['model'] == 4, 'lgb_penta_model_predictions']
df_test['lgb_multi_model'] = df_test['lgb_multi_model'].astype(np.float32)
 df_test['xgb_multi_model'] = 0
df_test.loc[df_test['model'] == 0, 'xgb_multi_model'] = df_test.loc[df_test['model'] == 0, 'xgb_duo_model_predictions']
df_test.loc[df_test['model'] == 1, 'xgb_multi_model'] = df_test.loc[df_test['model'] == 1, 'xgb_duo_model_predictions']
df_test.loc[df_test['model'] == 2, 'xgb_multi_model'] = df_test.loc[df_test['model'] == 2, 'xgb_trio_model_predictions']
df_test.loc[df_test['model'] == 3, 'xgb_multi_model'] = df_test.loc[df_test['model'] == 3, 'xgb_quad_model_predictions']
df_test.loc[df_test['model'] == 4, 'xgb_multi_model'] = np.nan
df_test['xgb_multi_model'] = df_test['xgb_multi_model'].astype(np.float32)
 df_test['cb_multi_model'] = 0
df_test.loc[df_test['model'] == 0, 'cb_multi_model'] = df_test.loc[df_test['model'] == 0, 'cb_duo_model_predictions']
df_test.loc[df_test['model'] == 1, 'cb_multi_model'] = df_test.loc[df_test['model'] == 1, 'cb_duo_model_predictions']
df_test.loc[df_test['model'] == 2, 'cb_multi_model'] = df_test.loc[df_test['model'] == 2, 'cb_trio_model_predictions']
df_test.loc[df_test['model'] == 3, 'cb_multi_model'] = df_test.loc[df_test['model'] == 3, 'cb_quad_model_predictions']
df_test.loc[df_test['model'] == 4, 'cb_multi_model'] = df_test.loc[df_test['model'] == 4, 'cb_penta_model_predictions']
df_test['cb_multi_model'] = df_test['cb_multi_model'].astype(np.float32)
 PREDICTION_COLS = ['lgb_solo_model', 'xgb_solo_model', 'cb_solo_model', 'lgb_multi_model', 'xgb_multi_model', 'cb_multi_model']
df_train[PREDICTION_COLS].to_pickle('train_predictions.pkl')
 df_test[PREDICTION_COLS].to_pickle('test_predictions.pkl' )<save_to_csv>
def add_meta_feature(path,df):
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
pet_id = df['PetID']
for pet in pet_id:
try:
with open(path + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 df.loc[:, 'vertex_x'] = vertex_xs
df.loc[:, 'vertex_y'] = vertex_ys
df.loc[:, 'bounding_confidence'] = bounding_confidences
df.loc[:, 'bounding_importance'] = bounding_importance_fracs
df.loc[:, 'dominant_blue'] = dominant_blues
df.loc[:, 'dominant_green'] = dominant_greens
df.loc[:, 'dominant_red'] = dominant_reds
df.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
df.loc[:, 'dominant_score'] = dominant_scores
df.loc[:, 'label_description'] = label_descriptions
df.loc[:, 'label_score'] = label_scores
return df
if MODEL_USE == 3 or MODEL_USE==0:
train_with_meta = add_meta_feature('.. /input/train_metadata/', train_with_desc)
 target_train = train_with_meta['AdoptionSpeed']
cleaned_train = train_with_meta.drop(columns=['Name', 'RescuerID', 'Description', 'PetID', 'AdoptionSpeed', 'label_description'])
 cleaned_train.fillna(0.0,inplace=True)
 test_with_meta = add_meta_feature('.. /input/test_metadata/', test_with_desc)
 test_pet_ID = test_with_desc['PetID']
test_X = test_with_meta.drop(columns=['Name', 'RescuerID', 'Description', 'PetID', 'label_description'])
 test_X.fillna(0.0, inplace=True)
 x_train, x_valid, y_train, y_valid = train_test_split(cleaned_train,
target_train,
test_size=0.2,
random_state=seed)
submission = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv')
 submission['open_channels'] = df_test['blend_predictions'].copy()
submission.to_csv('gbdt_blend_submission.csv', index=False, float_format='%.4f')
 print(f'GBDT Blend Submission
{submission["open_channels"].describe() }' )<set_options>
if MODEL_USE == 3 or MODEL_USE==0:
third_model = EnsembleModel(balancing=True)
 third_model.set_scorer(kappa)
 third_model.tune_best_param(x_train, y_train)
 third_model.validate(x_valid,y_valid )
sns.set_context("talk")
 style.use('fivethirtyeight')
<feature_engineering>
model = None
if MODEL_USE == 1:
model = first_model
if MODEL_USE == 2:
model = second_model
if MODEL_USE == 0 or MODEL_USE == 3:
pass
model = third_model
train['batches'] =(train.index // 500_000)+ 1 
<prepare_x_and_y>
model.re_fit_with_best_param(cleaned_train,target_train )
KNN = 100
batch = 1000
test_pred = np.zeros(( test.shape[0]),dtype=np.int8)
 for g in [0,1,2,3,4]:
print('Infering group %i'%g)
 data = train.loc[train.group==g]
X_train = np.zeros(( len(data)-6,7))
X_train[:,0] = 0.25*data.signal[:-6]
X_train[:,1] = 0.5*data.signal[1:-5]
X_train[:,2] = 1.0*data.signal[2:-4]
X_train[:,3] = 4.0*data.signal[3:-3]
X_train[:,4] = 1.0*data.signal[4:-2]
X_train[:,5] = 0.5*data.signal[5:-1]
X_train[:,6] = 0.25*data.signal[6:]
y_train = data.open_channels[3:].values
data = test.loc[test.group==g]
X_test = np.zeros(( len(data)-6,7))
X_test[:,0] = 0.25*data.signal[:-6]
X_test[:,1] = 0.5*data.signal[1:-5]
X_test[:,2] = 1.0*data.signal[2:-4]
X_test[:,3] = 4.0*data.signal[3:-3]
X_test[:,4] = 1.0*data.signal[4:-2]
X_test[:,5] = 0.5*data.signal[5:-1]
X_test[:,6] = 0.25*data.signal[6:]
model = NearestNeighbors(n_neighbors=KNN)
 model.fit(X_train)
 distances, indices = model.kneighbors(X_test)
 ct = indices.shape[0]
pred = np.zeros(( ct+6),dtype=np.int8)
 it = ct//batch + int(ct%batch!=0)
 print('Processing %i batches:'%(it))
for k in range(it):
a = batch*k; b = batch*(k+1); b = min(ct,b)
 pred[a+3:b+3] = np.median(y_train[ indices[a:b].astype(int)], axis=1)
 test_pred[test.group==g] = pred<predict_on_test>
final_result = model.predict(test_X )
<save_to_csv>
submission_df = pd.DataFrame(data={'PetID' : test_pet_ID.tolist() ,
'AdoptionSpeed' : final_result})
 submission_df.head(5 )
sub = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv')
 sub.open_channels = test_pred
sub.to_csv('submission.csv',index=False,float_format='%.4f')
 res=200
plt.figure(figsize=(20,5))
plt.plot(sub.time[::res],sub.open_channels[::res])
 plt.show()<import_modules>
submission_df.to_csv('submission.csv', index=False )