Kaggle competition: University of Liverpool - Ion Switching
<SOS> metric: MacroFScore Kaggle data source: university-of-liverpool-ion-switching<concatenate>
warnings.filterwarnings('ignore')
 pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000)
 LGB_SEED = 1337
XGB_SEED = 5041992
CB_SEED = 7021991
print(f'LightGBM Version: {lgb.__version__} - Seed: {LGB_SEED}')
 print(f'XGBoost Version: {xgb.__version__} - Seed: {XGB_SEED}')
 print(f'CatBoost Version: {cb.__version__} - Seed: {CB_SEED}' )
full_data=pd.concat([train,test])
 full_data['text']=full_data['text'].str.lower()
full_data.shape<feature_engineering>
df_train = pd.read_pickle('.. /input/ion-switching-preprocessing/train.pkl')
 df_test = pd.read_pickle('.. /input/ion-switching-preprocessing/test.pkl')
 df_test.loc[df_test['model'] == 1.5, 'model'] = 2
df_train['is_filtered'] = df_train['is_filtered'].astype(np.uint8)
 df_train['model'] = df_train['model'].astype(np.uint8)
 df_test['model'] = df_test['model'].astype(np.uint8)
 for df in [df_train, df_test]:
df.drop(columns=['signal', 'time_scaled', 'signal_processed_kalman', 'batch'], inplace=True)
 df_train['signal_processed_original'] = df_train['signal_processed']
df_test['signal_processed_original'] = df_test['signal_processed']
df_train['signal_processed'] = df_train['signal_processed_denoised']
df_test['signal_processed'] = df_test['signal_processed_denoised']
for df in [df_train, df_test]:
df.drop(columns=['signal_processed_denoised'], inplace=True)
 print(f'Training Set Shape: {df_train.shape}')
 print(f'Test Set Shape: {df_test.shape}')
 print(f'Training Set Memory Usage: {df_train.memory_usage().sum() / 1024 ** 2:.2f} MB')
 print(f'Test Set Memory Usage: {df_test.memory_usage().sum() / 1024 ** 2:.2f} MB' )
full_data['text_punct']=full_data['text'].apply(lambda x:find_punct(x))
full_data['text_punct_len']=full_data['text'].apply(lambda x:len(find_punct(x)) )<create_dataframe>
class MultiModelGBDTEnsemble:
def __init__(self,lgb_parameters, xgb_parameters, cb_parameters, model):
self.K = 2
self.kf = KFold(n_splits=self.K)
 self.model = model
self.lgb_parameters = lgb_parameters
self.xgb_parameters = xgb_parameters
self.cb_parameters = cb_parameters
def _lightgbm_train_and_predict(self, X_train, y_train, X_test, weight, model):
print(f'
self.lgb_scores = []
self.lgb_oof = np.zeros(X_train.shape[0])
 self.lgb_y_pred = pd.DataFrame(np.zeros(( X_test.shape[0], self.K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, self.K + 1)])
 self.lgb_importance = pd.DataFrame(np.zeros(( X_train.shape[1], self.K)) , columns=[f'Fold_{i}_Importance' for i in range(1, self.K + 1)], index=X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(self.kf.split(X_train, y_train), 1):
print(f'
Fold {fold}')
 lgb_trn_data = lgb.Dataset(X_train.iloc[trn_idx, :], label=y_train.iloc[trn_idx], weight=weight[trn_idx])
 lgb_val_data = lgb.Dataset(X_train.iloc[val_idx, :], label=y_train.iloc[val_idx])
 lgb_model = lgb.train(self.lgb_parameters, lgb_trn_data, valid_sets=[lgb_trn_data, lgb_val_data], verbose_eval=50)
 lgb_oof_predictions = lgb_model.predict(X_train.iloc[val_idx, :], num_iteration=lgb_model.best_iteration)
 self.lgb_oof[val_idx] = lgb_oof_predictions
df_train.loc[X_train.iloc[val_idx, :].index, 'lgb_solo_model_oof'] = lgb_oof_predictions
lgb_test_predictions = lgb_model.predict(X_test, num_iteration=lgb_model.best_iteration)
 self.lgb_y_pred.iloc[:, fold - 1] = lgb_test_predictions
df_test.loc[X_test.index, f'fold{fold}_lgb_solo_model_predictions'] = lgb_test_predictions
self.lgb_importance.iloc[:, fold - 1] = lgb_model.feature_importance(importance_type='gain')
 lgb_score = f1_score(y_train.iloc[val_idx].values, np.round(np.clip(lgb_oof_predictions, y_train.min() , y_train.max())) , average='macro')
 self.lgb_scores.append(lgb_score)
 print('
Solo LGB Fold {} Macro F1-Score {}
'.format(fold, lgb_score))
print('--------------------')
 print(f'Solo LGB Mean Macro F1-Score {np.mean(self.lgb_scores):.6} [STD:{np.std(self.lgb_scores):.6}]')
 print(f'Solo LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_train, np.round(np.clip(self.lgb_oof, y_train.min() , y_train.max())) , average="macro"):.6}')
 print('--------------------')
 def _xgboost_train_and_predict(self, X_train, y_train, X_test, num_boosting_round, early_stopping_rounds, weight, model):
print(f'
self.xgb_scores = []
self.xgb_oof = np.zeros(X_train.shape[0])
 self.xgb_y_pred = pd.DataFrame(np.zeros(( X_test.shape[0], self.K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, self.K + 1)])
 self.xgb_importance = pd.DataFrame(np.zeros(( X_train.shape[1], self.K)) , columns=[f'Fold_{i}_Importance' for i in range(1, self.K + 1)], index=X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(self.kf.split(X_train, y_train), 1):
print(f'
Fold {fold}')
 xgb_trn_data = xgb.DMatrix(X_train.iloc[trn_idx, :], label=y_train.iloc[trn_idx], weight=weight[trn_idx])
 xgb_val_data = xgb.DMatrix(X_train.iloc[val_idx, :], label=y_train.iloc[val_idx])
 xgb_model = xgb.train(self.xgb_parameters, xgb_trn_data, num_boosting_round, evals=[(xgb_trn_data, 'train'),(xgb_val_data, 'val')], verbose_eval=50, early_stopping_rounds=early_stopping_rounds)
 xgb_oof_predictions = xgb_model.predict(xgb.DMatrix(X_train.iloc[val_idx, :]))
self.xgb_oof[val_idx] = xgb_oof_predictions
df_train.loc[X_train.iloc[val_idx, :].index, 'xgb_solo_model_oof'] = xgb_oof_predictions
xgb_test_predictions = xgb_model.predict(xgb.DMatrix(X_test))
self.xgb_y_pred.iloc[:, fold - 1] = xgb_test_predictions
df_test.loc[X_test.index, f'fold{fold}_xgb_solo_model_predictions'] = xgb_test_predictions
self.xgb_importance.iloc[:, fold - 1] = list(xgb_model.get_score(importance_type='gain' ).values())
 xgb_score = f1_score(y_train.iloc[val_idx].values, np.round(np.clip(xgb_oof_predictions, y_train.min() , y_train.max())) , average='macro')
 self.xgb_scores.append(xgb_score)
 print('Solo XGB Fold {} Macro F1-Score {}
'.format(fold, xgb_score))
print('--------------------')
 print(f'Solo XGB Mean Macro F1-Score {np.mean(self.xgb_scores):.6} [STD:{np.std(self.xgb_scores):.6}]')
 print(f'Solo XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_train, np.round(np.clip(self.xgb_oof, y_train.min() , y_train.max())) , average="macro"):.6}')
 print('--------------------')
 def _catboost_train_and_predict(self, X_train, y_train, X_test, weight, model):
print(f'
self.cb_scores = []
self.cb_oof = np.zeros(X_train.shape[0])
 self.cb_y_pred = pd.DataFrame(np.zeros(( X_test.shape[0], self.K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, self.K + 1)])
 self.cb_importance = pd.DataFrame(np.zeros(( X_train.shape[1], self.K)) , columns=[f'Fold_{i}_Importance' for i in range(1, self.K + 1)], index=X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(self.kf.split(X_train, y_train), 1):
print(f'
Fold {fold}')
 cb_trn_data = cb.Pool(X_train.iloc[trn_idx, :], label=y_train.iloc[trn_idx], weight=weight[trn_idx])
 cb_val_data = cb.Pool(X_train.iloc[val_idx, :], label=y_train.iloc[val_idx])
 cb_model = cb.CatBoostRegressor(**self.cb_parameters)
 cb_model.fit(cb_trn_data)
 cb_oof_predictions = cb_model.predict(cb_val_data)
 self.cb_oof[val_idx] = cb_oof_predictions
df_train.loc[X_train.iloc[val_idx, :].index, 'cb_solo_model_oof'] = cb_oof_predictions
cb_test_predictions = cb_model.predict(cb.Pool(X_test))
self.cb_y_pred.iloc[:, fold - 1] = cb_test_predictions
df_test.loc[X_test.index, f'fold{fold}_cb_solo_model_predictions'] = cb_test_predictions
self.cb_importance.iloc[:, fold - 1] = cb_model.get_feature_importance()
cb_score = f1_score(y_train.iloc[val_idx].values, np.round(np.clip(cb_oof_predictions, y_train.min() , y_train.max())) , average='macro')
 self.cb_scores.append(cb_score)
 print('
Solo CB Fold {} Macro F1-Score {}
'.format(fold, cb_score))
print('--------------------')
 print(f'Solo CB Mean Macro F1-Score {np.mean(self.cb_scores):.6} [STD:{np.std(self.cb_scores):.6}]')
 print(f'Solo CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_train, np.round(np.clip(self.cb_oof, y_train.min() , y_train.max())) , average="macro"):.6}')
 print('--------------------')
 def train_and_predict(self,
lgb_X_train, lgb_y_train, lgb_X_test,
xgb_X_train, xgb_y_train, xgb_X_test,
cb_X_train, cb_y_train, cb_X_test,
num_boosting_round=100, early_stopping_rounds=25,
lgb_sample_weight=None, xgb_sample_weight=None, cb_sample_weight=None, model=None):
if self.model == 'LightGBM':
self._lightgbm_train_and_predict(lgb_X_train, lgb_y_train, lgb_X_test, weight=lgb_sample_weight, model=model)
 elif self.model == 'XGBoost':
self._xgboost_train_and_predict(xgb_X_train, xgb_y_train, xgb_X_test, num_boosting_round=num_boosting_round, early_stopping_rounds=early_stopping_rounds, weight=xgb_sample_weight, model=model)
 elif self.model == 'CatBoost':
self._catboost_train_and_predict(cb_X_train, cb_y_train, cb_X_test, weight=cb_sample_weight, model=model)
 elif self.model == 'Blend':
self._lightgbm_train_and_predict(lgb_X_train, lgb_y_train, lgb_X_test, weight=lgb_sample_weight, model=model)
 self._xgboost_train_and_predict(xgb_X_train, xgb_y_train, xgb_X_test, num_boosting_round=num_boosting_round, early_stopping_rounds=early_stopping_rounds, weight=xgb_sample_weight, model=model)
 self._catboost_train_and_predict(cb_X_train, cb_y_train, cb_X_test, weight=cb_sample_weight, model=model)
 self.blend_oof =(self.lgb_oof * 0.34)+(self.xgb_oof * 0.33)+(self.cb_oof * 0.33)
 print(f'Solo Model {model} Equal Blend OOF(Rounded)Macro F1-Score {f1_score(lgb_y_train, np.round(np.clip(self.blend_oof, lgb_y_train.min() , lgb_y_train.max())) , average="macro")}
')
 def plot_importance(self):
if self.model == 'Blend':
self.lgb_importance['Mean_Importance'] = self.lgb_importance.sum(axis=1)/ self.K
self.lgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 self.xgb_importance['Mean_Importance'] = self.xgb_importance.sum(axis=1)/ self.K
self.xgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 self.cb_importance['Mean_Importance'] = self.cb_importance.sum(axis=1)/ self.K
self.cb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 fig, axs = plt.subplots(nrows=3, figsize=(20, 40))
sns.barplot(x='Mean_Importance', y=self.lgb_importance.index, data=self.lgb_importance, ax=axs[0])
 sns.barplot(x='Mean_Importance', y=self.xgb_importance.index, data=self.xgb_importance, ax=axs[1])
 sns.barplot(x='Mean_Importance', y=self.cb_importance.index, data=self.cb_importance, ax=axs[2])
 for i in range(3):
axs[i].set_xlabel('')
 axs[i].tick_params(axis='x', labelsize=20)
 axs[i].tick_params(axis='y', labelsize=20)
 axs[0].set_title(f'Solo LightGBM Feature Importance(Gain)', size=20, pad=20)
 axs[1].set_title(f'Solo XGBoost Feature Importance(Gain)', size=20, pad=20)
 axs[2].set_title(f'Solo CatBoost Feature Importance(Gain)', size=20, pad=20)
 else:
plt.figure(figsize=(20, 6))
if self.model == 'LightGBM':
self.lgb_importance['Mean_Importance'] = self.lgb_importance.sum(axis=1)/ self.K
self.lgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=self.lgb_importance.index, data=self.lgb_importance)
 elif self.model == 'XGBoost':
self.xgb_importance['Mean_Importance'] = self.xgb_importance.sum(axis=1)/ self.K
self.xgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=self.xgb_importance.index, data=self.xgb_importance)
 elif self.model == 'CatBoost':
self.cb_importance['Mean_Importance'] = self.cb_importance.sum(axis=1)/ self.K
self.cb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=self.cb_importance.index, data=self.cb_importance)
 if self.model != 'Blend':
plt.xlabel('')
 plt.tick_params(axis='x', labelsize=20)
 plt.tick_params(axis='y', labelsize=20)
 plt.title(f'Solo {self.model} Feature Importance(Gain)', size=20, pad=20)
 plt.show()
punc_text_df=pd.DataFrame(full_data,columns=['text_punct','sentiment'])
 punc_text_df=punc_text_df[punc_text_df['text_punct'].map(lambda d: len(d)) > 0]
punc_text_df=punc_text_df.explode('text_punct')
 positive_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="positive"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'pos_punct'})
 negative_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="negative"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'neg_punct'})
 neutral_text_df=pd.DataFrame(punc_text_df.loc[punc_text_df['sentiment']=="neutral"]['text_punct'].value_counts() ).reset_index().rename(columns={'index': 'punct','text_punct':'neut_punct'})
<feature_engineering>
print('
print('------------------------------------------------------')
 model0_lgb_features = ['signal_processed']
model0_lgb_X_train = df_train[df_train['model'] == 0][model0_lgb_features].copy(deep=True)
 model0_lgb_y_train = df_train[df_train['model'] == 0]['open_channels'].copy(deep=True)
 model0_lgb_X_test = df_test[df_test['model'] == 0][model0_lgb_features].copy(deep=True)
 model0_lgb_sample_weight = model0_lgb_y_train.copy(deep=True ).reset_index(drop=True)
 model0_lgb_sample_weight.loc[model0_lgb_sample_weight == 0] = 1
model0_lgb_sample_weight.loc[model0_lgb_sample_weight == 1] = 1
for df in [model0_lgb_X_train, model0_lgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_mean_50'] = df['signal_processed'].rolling(window=50, min_periods=1, center=False ).mean()
df['signal_processed_rolling_forward_center_std_20'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_mean_70_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=70, min_periods=1, center=False ).mean()
df['signal_processed_rolling_forward_center_max_25_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=25, min_periods=1, center=True ).max()
df['signal_processed_diff+1'] = df['signal_processed'].diff(1)
 df['signal_processed_rolling_forward_center_min_35'] = df['signal_processed'].rolling(window=35, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_35'] = df['signal_processed'].rolling(window=35, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_center_maxmin_ratio_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).max() / df['signal_processed'].rolling(window=15, min_periods=1, center=False ).min()
model0_lgb_parameters = {
'num_iterations': 250,
'early_stopping_round': 50,
'num_leaves': 2 ** 7,
'learning_rate': 0.075,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 0.5,
'feature_fraction_bynode': 0.5,
'min_data_in_leaf': 20,
'lambda_l1': 0.5,
'lambda_l2': 0,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model0_xgb_features = ['signal_processed']
model0_xgb_X_train = df_train[df_train['model'] == 0][model0_xgb_features].copy(deep=True)
 model0_xgb_y_train = df_train[df_train['model'] == 0]['open_channels'].copy(deep=True)
 model0_xgb_X_test = df_test[df_test['model'] == 0][model0_xgb_features].copy(deep=True)
 model0_xgb_sample_weight = model0_xgb_y_train.copy(deep=True ).reset_index(drop=True)
 model0_xgb_sample_weight.loc[model0_xgb_sample_weight == 0] = 1
model0_xgb_sample_weight.loc[model0_xgb_sample_weight == 1] = 1
for df in [model0_xgb_X_train, model0_xgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_rolling_forward_mean_50'] = df['signal_processed'].rolling(window=50, min_periods=1, center=False ).mean()
df['signal_processed_rolling_forward_center_std_45'] = df['signal_processed'].rolling(window=45, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_min_45'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_45'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_center_maxminrange_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).max() - df['signal_processed'].rolling(window=30, min_periods=1, center=True ).min()
df['signal_processed_diff+1'] = df['signal_processed'].diff(1 ).fillna(0)
 df['signal_processed_rolling_forward_center_mean_50_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=20, min_periods=1, center=False ).mean()
model0_xgb_parameters = {
'n_estimators': 2 ** 8,
'learning_rate': 0.05,
'colsample_bytree': 0.8,
'colsample_bylevel': 0.8,
'colsample_bynode': 0.8,
'sumbsample': 0.9,
'max_depth': 5,
'gamma': 0.75,
'min_child_weight': 1,
'lambda': 1,
'alpha': 0.1,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model0_cb_features = ['signal_processed']
model0_cb_X_train = df_train[df_train['model'] == 0][model0_cb_features].copy(deep=True)
 model0_cb_y_train = df_train[df_train['model'] == 0]['open_channels'].copy(deep=True)
 model0_cb_X_test = df_test[df_test['model'] == 0][model0_cb_features].copy(deep=True)
 model0_cb_sample_weight = model0_cb_y_train.copy(deep=True ).reset_index(drop=True)
 model0_cb_sample_weight.loc[model0_cb_sample_weight == 0] = 1
model0_cb_sample_weight.loc[model0_cb_sample_weight == 1] = 1
for df in [model0_cb_X_train, model0_cb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_rolling_forward_center_mean_40'] = df['signal_processed'].rolling(window=40, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_min_45'] = df['signal_processed'].rolling(window=45, min_periods=1, center=False ).min()
df['signal_processed_rolling_forward_center_max_50'] = df['signal_processed'].rolling(window=50, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_center_maxminratio_10'] = df['signal_processed'].rolling(window=25, min_periods=1, center=True ).max() / df['signal_processed'].rolling(window=25, min_periods=1, center=True ).min()
model0_cb_parameters = {
'n_estimators': 250,
'learning_rate': 0.06,
'depth': 6,
'subsample': 0.7,
'bagging_temperature': 1,
'colsample_bylevel': 0.6,
'l2_leaf_reg': 0.04,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
solo_model0 = MultiModelGBDTEnsemble(lgb_parameters=model0_lgb_parameters,
xgb_parameters=model0_xgb_parameters,
cb_parameters=model0_cb_parameters,
model='Blend')
 solo_model0.train_and_predict(lgb_X_train=model0_lgb_X_train, lgb_y_train=model0_lgb_y_train, lgb_X_test=model0_lgb_X_test,
xgb_X_train=model0_xgb_X_train, xgb_y_train=model0_xgb_y_train, xgb_X_test=model0_xgb_X_test,
cb_X_train=model0_cb_X_train, cb_y_train=model0_cb_y_train, cb_X_test=model0_cb_X_test,
num_boosting_round=300, early_stopping_rounds=15,
lgb_sample_weight=model0_lgb_sample_weight, xgb_sample_weight=model0_xgb_sample_weight, cb_sample_weight=model0_cb_sample_weight,
model=0)
 solo_model0.plot_importance()
del model0_lgb_X_train, model0_lgb_y_train, model0_lgb_X_test, model0_lgb_sample_weight, model0_lgb_parameters
del model0_xgb_X_train, model0_xgb_y_train, model0_xgb_X_test, model0_xgb_sample_weight, model0_xgb_parameters
del model0_cb_X_train, model0_cb_y_train, model0_cb_X_test, model0_cb_sample_weight, model0_cb_parameters
del solo_model0
full_data['text']=full_data['text'].apply(lambda x:remove_link(x))
full_data['text']=full_data['text'].apply(lambda x:remove_punct(x))<filter>
print('
print('------------------------------------------------------')
 model1_lgb_features = ['signal_processed']
model1_lgb_X_train = df_train[df_train['model'] == 1][model1_lgb_features].copy(deep=True)
 model1_lgb_y_train = df_train[df_train['model'] == 1]['open_channels'].copy(deep=True)
 model1_lgb_X_test = df_test[df_test['model'] == 1][model1_lgb_features].copy(deep=True)
 model1_lgb_sample_weight = model1_lgb_y_train.copy(deep=True ).reset_index(drop=True)
 model1_lgb_sample_weight.loc[model1_lgb_sample_weight == 0] = 1
model1_lgb_sample_weight.loc[model1_lgb_sample_weight == 1] = 1
for df in [model1_lgb_X_train, model1_lgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_mean_45'] = df['signal_processed'].rolling(window=45, min_periods=1, center=False ).mean()
df['signal_processed_rolling_forward_std_10'] = df['signal_processed'].rolling(window=20, min_periods=1, center=False ).std()
df['signal_processed_rolling_forward_center_min_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_center_maxmindifference_10'] = df['signal_processed'].rolling(window=5, min_periods=1, center=True ).max() - df['signal_processed'].rolling(window=5, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_min_15_shift-15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).min().shift(-15)
 df['signal_processed_rolling_forward_max_15_shift-15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).max().shift(-15)
 df['signal_processed_rolling_forward_mean_15_shift-15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).mean().shift(-15)
 model1_lgb_parameters = {
'num_iterations': 250,
'early_stopping_round': 50,
'num_leaves': 2 ** 5,
'learning_rate': 0.085,
'bagging_fraction': 0.5,
'bagging_freq': 1,
'feature_fraction': 0.7,
'feature_fraction_bynode': 0.5,
'min_data_in_leaf': 20,
'lambda_l1': 0.1,
'lambda_l2': 0.5,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model1_xgb_features = ['signal_processed']
model1_xgb_X_train = df_train[df_train['model'] == 1][model1_xgb_features].copy(deep=True)
 model1_xgb_y_train = df_train[df_train['model'] == 1]['open_channels'].copy(deep=True)
 model1_xgb_X_test = df_test[df_test['model'] == 1][model1_xgb_features].copy(deep=True)
 model1_xgb_sample_weight = model1_xgb_y_train.copy(deep=True ).reset_index(drop=True)
 model1_xgb_sample_weight.loc[model1_xgb_sample_weight == 0] = 1
model1_xgb_sample_weight.loc[model1_xgb_sample_weight == 1] = 1
for df in [model1_xgb_X_train, model1_xgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_median_25'] = df['signal_processed'].rolling(window=25, min_periods=1, center=False ).median()
df['signal_processed_rolling_forward_std_10'] = df['signal_processed'].rolling(window=25, min_periods=1, center=False ).std()
df['signal_processed_rolling_forward_center_min_10'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=False ).max()
df['signal_processed_rolling_forward_center_maxminratio_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).max() / df['signal_processed'].rolling(window=15, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_min_15_shift-15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).min().shift(-15)
 df['signal_processed_rolling_forward_max_15_shift-15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).max().shift(-15)
 df['signal_processed_rolling_forward_mean_15_shift-15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).mean().shift(-15)
 model1_xgb_parameters = {
'n_estimators': 2 ** 6,
'learning_rate': 0.085,
'colsample_bytree': 0.8,
'colsample_bylevel': 0.8,
'colsample_bynode': 0.7,
'sumbsample': 0.8,
'max_depth': 6,
'gamma': 0.4,
'min_child_weight': 1,
'lambda': 0.6,
'alpha': 0,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model1_cb_features = ['signal_processed']
model1_cb_X_train = df_train[df_train['model'] == 1][model1_cb_features].copy(deep=True)
 model1_cb_y_train = df_train[df_train['model'] == 1]['open_channels'].copy(deep=True)
 model1_cb_X_test = df_test[df_test['model'] == 1][model1_cb_features].copy(deep=True)
 model1_cb_sample_weight = model1_cb_y_train.copy(deep=True ).reset_index(drop=True)
 model1_cb_sample_weight.loc[model1_cb_sample_weight == 0] = 1
model1_cb_sample_weight.loc[model1_cb_sample_weight == 1] = 1
for df in [model1_cb_X_train, model1_cb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_45'] = df['signal_processed'].rolling(window=50, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_std_20'] = df['signal_processed'].rolling(window=20, min_periods=1, center=False ).std()
df['signal_processed_rolling_forward_min_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=False ).min()
df['signal_processed_rolling_forward_center_max_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_min_15_shift-15'] = df['signal_processed'].rolling(window=10 ).min().shift(-10)
 df['signal_processed_rolling_forward_max_15_shift-15'] = df['signal_processed'].rolling(window=10 ).max().shift(-10)
 df['signal_processed_rolling_forward_mean_15_shift-15'] = df['signal_processed'].rolling(window=10 ).mean().shift(-10)
 model1_cb_parameters = {
'n_estimators': 200,
'learning_rate': 0.055,
'depth': 6,
'subsample': 0.6,
'bagging_temperature': 1,
'colsample_bylevel': 0.9,
'l2_leaf_reg': 0.5,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
solo_model1 = MultiModelGBDTEnsemble(lgb_parameters=model1_lgb_parameters,
xgb_parameters=model1_xgb_parameters,
cb_parameters=model1_cb_parameters,
model='Blend')
 solo_model1.train_and_predict(lgb_X_train=model1_lgb_X_train, lgb_y_train=model1_lgb_y_train, lgb_X_test=model1_lgb_X_test,
xgb_X_train=model1_xgb_X_train, xgb_y_train=model1_xgb_y_train, xgb_X_test=model1_xgb_X_test,
cb_X_train=model1_cb_X_train, cb_y_train=model1_cb_y_train, cb_X_test=model1_cb_X_test,
num_boosting_round=250, early_stopping_rounds=10,
lgb_sample_weight=model1_lgb_sample_weight, xgb_sample_weight=model1_xgb_sample_weight, cb_sample_weight=model1_cb_sample_weight,
model=1)
 solo_model1.plot_importance()
del model1_lgb_X_train, model1_lgb_y_train, model1_lgb_X_test, model1_lgb_sample_weight, model1_lgb_parameters
del model1_xgb_X_train, model1_xgb_y_train, model1_xgb_X_test, model1_xgb_sample_weight, model1_xgb_parameters
del model1_cb_X_train, model1_cb_y_train, model1_cb_X_test, model1_cb_sample_weight, model1_cb_parameters
del solo_model1
full_data.loc[full_data['text']=="",['text']]="nothing"<prepare_x_and_y>
print('
print('------------------------------------------------------')
 model2_lgb_features = ['signal_processed']
model2_lgb_X_train = df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)][model2_lgb_features].copy(deep=True)
 model2_lgb_y_train = df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model2_lgb_X_test = df_test[df_test['model'] == 2][model2_lgb_features].copy(deep=True)
 model2_lgb_sample_weight = model2_lgb_y_train.copy(deep=True ).reset_index(drop=True)
 model2_lgb_sample_weight.loc[model2_lgb_sample_weight == 0] = 1
model2_lgb_sample_weight.loc[model2_lgb_sample_weight == 1] = 1
model2_lgb_sample_weight.loc[model2_lgb_sample_weight == 2] = 1
model2_lgb_sample_weight.loc[model2_lgb_sample_weight == 3] = 1
for df in [model2_lgb_X_train, model2_lgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_squarerooted'] = df['signal_processed'] **(1 / 2)
 df['signal_processed_rolling_forward_center_mean_45'] = df['signal_processed'].rolling(window=45, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_median_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).median()
df['signal_processed_rolling_forward_center_std_20'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_min_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_35'] = df['signal_processed'].rolling(window=35, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_center_median_50_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=50, min_periods=1, center=True ).median()
df['signal_processed_rolling_forward_min_30_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=30, min_periods=1, center=False ).min()
df['signal_processed_rolling_forward_maxminratio_50'] = df['signal_processed'].rolling(window=50, min_periods=1, center=False ).max() / df['signal_processed'].rolling(window=50, min_periods=1, center=False ).min()
df['signal_processed_diff+1_rolling_forward_center_mean_20'] = df['signal_processed'].diff(1 ).fillna(0 ).rolling(window=20, min_periods=1, center=True ).mean()
model2_lgb_parameters = {
'num_iterations': 250,
'early_stopping_round': 50,
'num_leaves': 2 ** 7,
'learning_rate': 0.05,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 0.9,
'feature_fraction_bynode': 0.5,
'min_data_in_leaf': 20,
'lambda_l1': 0,
'lambda_l2': 0.05,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model2_xgb_features = ['signal_processed']
model2_xgb_X_train = df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)][model2_xgb_features].copy(deep=True)
 model2_xgb_y_train = df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model2_xgb_X_test = df_test[(df_test['model'] == 2)|(df_test['model'] == 1.5)][model2_xgb_features].copy(deep=True)
 model2_xgb_sample_weight = model2_xgb_y_train.copy(deep=True ).reset_index(drop=True)
 model2_xgb_sample_weight.loc[model2_xgb_sample_weight == 0] = 1
model2_xgb_sample_weight.loc[model2_xgb_sample_weight == 1] = 1
model2_xgb_sample_weight.loc[model2_xgb_sample_weight == 2] = 1
model2_xgb_sample_weight.loc[model2_xgb_sample_weight == 3] = 1
for df in [model2_xgb_X_train, model2_xgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_rolling_forward_center_mean_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_std_25'] = df['signal_processed'].rolling(window=25, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_min_40'] = df['signal_processed'].rolling(window=40, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).max()
model2_xgb_parameters = {
'n_estimators': 2 ** 7,
'learning_rate': 0.09,
'colsample_bytree': 0.9,
'colsample_bylevel': 0.9,
'colsample_bynode': 0.8,
'sumbsample': 0.6,
'max_depth': 6,
'gamma': 0.01,
'min_child_weight': 1,
'lambda': 1,
'alpha': 0.005,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model2_cb_features = ['signal_processed']
model2_cb_X_train = df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)][model2_cb_features].copy(deep=True)
 model2_cb_y_train = df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model2_cb_X_test = df_test[(df_test['model'] == 2)|(df_test['model'] == 1.5)][model2_cb_features].copy(deep=True)
 model2_cb_sample_weight = model2_cb_y_train.copy(deep=True ).reset_index(drop=True)
 model2_cb_sample_weight.loc[model2_cb_sample_weight == 0] = 1
model2_cb_sample_weight.loc[model2_cb_sample_weight == 1] = 1
model2_cb_sample_weight.loc[model2_cb_sample_weight == 2] = 1
model2_cb_sample_weight.loc[model2_cb_sample_weight == 3] = 1
for df in [model2_cb_X_train, model2_cb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_rolling_forward_center_mean_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_std_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_min_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_35'] = df['signal_processed'].rolling(window=35, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_mean_30_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=30, min_periods=1, center=False ).mean()
df['signal_processed_rolling_forward_center_min_45_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=45, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_median_45'] = df['signal_processed'].rolling(window=45, min_periods=1, center=True ).median()
df['signal_processed_rolling_forward_min_25_shift-25'] = df['signal_processed'].rolling(window=25, min_periods=1, center=False ).min().shift(-25)
 df['signal_processed_rolling_forward_max_25_shift-25'] = df['signal_processed'].rolling(window=25, min_periods=1, center=False ).max().shift(-25)
 df['signal_processed_rolling_forward_mean_25_shift-25'] = df['signal_processed'].rolling(window=25, min_periods=1, center=False ).mean().shift(-25)
 df['signal_processed_fraction'] = np.abs(df['signal_processed'] - df['signal_processed'].astype(int))
df['signal_processed_squared'] = df['signal_processed'] ** 2
model2_cb_parameters = {
'n_estimators': 300,
'learning_rate': 0.08,
'depth': 6,
'subsample': 0.6,
'bagging_temperature': 1,
'colsample_bylevel': 0.8,
'l2_leaf_reg': 0.06,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
solo_model2 = MultiModelGBDTEnsemble(lgb_parameters=model2_lgb_parameters,
xgb_parameters=model2_xgb_parameters,
cb_parameters=model2_cb_parameters,
model='Blend')
 solo_model2.train_and_predict(lgb_X_train=model2_lgb_X_train, lgb_y_train=model2_lgb_y_train, lgb_X_test=model2_lgb_X_test,
xgb_X_train=model2_xgb_X_train, xgb_y_train=model2_xgb_y_train, xgb_X_test=model2_xgb_X_test,
cb_X_train=model2_cb_X_train, cb_y_train=model2_cb_y_train, cb_X_test=model2_cb_X_test,
num_boosting_round=300, early_stopping_rounds=25,
lgb_sample_weight=model2_lgb_sample_weight, xgb_sample_weight=model2_xgb_sample_weight, cb_sample_weight=model2_cb_sample_weight,
model=2)
 solo_model2.plot_importance()
del model2_lgb_X_train, model2_lgb_y_train, model2_lgb_X_test, model2_lgb_sample_weight, model2_lgb_parameters
del model2_xgb_X_train, model2_xgb_y_train, model2_xgb_X_test, model2_xgb_sample_weight, model2_xgb_parameters
del model2_cb_X_train, model2_cb_y_train, model2_cb_X_test, model2_cb_sample_weight, model2_cb_parameters
del solo_model2
def corpus_sentiment_stop(data,feature,sentiment):
corpus=create_corpus(data,feature,sentiment)
 dic=defaultdict(int)
 for word in corpus:
if word in stop:
dic[word]+=1
top=sorted(dic.items() , key=lambda x:x[1],reverse=True)
 x,y=zip(*top)
 return x,y 
<define_variables>
print('
print('------------------------------------------------------')
 model3_lgb_features = ['signal_processed']
model3_lgb_X_train = df_train[df_train['model'] == 3][model3_lgb_features].copy(deep=True)
 model3_lgb_y_train = df_train[df_train['model'] == 3]['open_channels'].copy(deep=True)
 model3_lgb_X_test = df_test[df_test['model'] == 3][model3_lgb_features].copy(deep=True)
 model3_lgb_sample_weight = model3_lgb_y_train.copy(deep=True ).reset_index(drop=True)
 model3_lgb_sample_weight.loc[model3_lgb_sample_weight == 0] = 1
model3_lgb_sample_weight.loc[model3_lgb_sample_weight == 1] = 1
model3_lgb_sample_weight.loc[model3_lgb_sample_weight == 2] = 1
model3_lgb_sample_weight.loc[model3_lgb_sample_weight == 3] = 1
model3_lgb_sample_weight.loc[model3_lgb_sample_weight == 4] = 1
model3_lgb_sample_weight.loc[model3_lgb_sample_weight == 5] = 1
for df in [model3_lgb_X_train, model3_lgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_rolling_forward_mean_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).mean()
df['signal_processed_rolling_forward_std_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=False ).std()
df['signal_processed_rolling_forward_center_min_20'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_35'] = df['signal_processed'].rolling(window=35, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_maxmindifference_35'] = df['signal_processed'].rolling(window=25, min_periods=1, center=False ).max() - df['signal_processed'].rolling(window=25, min_periods=1, center=False ).min()
df['signal_processed_rolling_forward_center_mean_15_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=15, min_periods=1, center=False ).mean()
df['signal_processed_pctchange-1'] = df['signal_processed'].pct_change(-1)
 model3_lgb_parameters = {
'num_iterations': 200,
'early_stopping_round': 50,
'num_leaves': 2 ** 7,
'learning_rate': 0.06,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 0.9,
'feature_fraction_bynode': 0.8,
'min_data_in_leaf': 20,
'lambda_l1': 0.5,
'lambda_l2': 0,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model3_xgb_features = ['signal_processed']
model3_xgb_X_train = df_train[df_train['model'] == 3][model3_xgb_features].copy(deep=True)
 model3_xgb_y_train = df_train[df_train['model'] == 3]['open_channels'].copy(deep=True)
 model3_xgb_X_test = df_test[df_test['model'] == 3][model3_xgb_features].copy(deep=True)
 model3_xgb_sample_weight = model3_xgb_y_train.copy(deep=True ).reset_index(drop=True)
 model3_xgb_sample_weight.loc[model3_xgb_sample_weight == 0] = 1
model3_xgb_sample_weight.loc[model3_xgb_sample_weight == 1] = 1
model3_xgb_sample_weight.loc[model3_xgb_sample_weight == 2] = 1
model3_xgb_sample_weight.loc[model3_xgb_sample_weight == 3] = 1
model3_xgb_sample_weight.loc[model3_xgb_sample_weight == 4] = 1
model3_xgb_sample_weight.loc[model3_xgb_sample_weight == 5] = 1
for df in [model3_xgb_X_train, model3_xgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_rolling_forward_center_mean_35'] = df['signal_processed'].rolling(window=35, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_std_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_min_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_mean_45_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=45, min_periods=1, center=False ).mean()
model3_xgb_parameters = {
'n_estimators': 2 ** 7,
'learning_rate': 0.09,
'colsample_bytree': 0.9,
'colsample_bylevel': 0.9,
'colsample_bynode': 0.9,
'sumbsample': 0.6,
'max_depth': 6,
'gamma': 0,
'min_child_weight': 1,
'lambda': 1,
'alpha': 0,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model3_cb_features = ['signal_processed']
model3_cb_X_train = df_train[df_train['model'] == 3][model3_cb_features].copy(deep=True)
 model3_cb_y_train = df_train[df_train['model'] == 3]['open_channels'].copy(deep=True)
 model3_cb_X_test = df_test[df_test['model'] == 3][model3_cb_features].copy(deep=True)
 model3_cb_sample_weight = model3_cb_y_train.copy(deep=True ).reset_index(drop=True)
 model3_cb_sample_weight.loc[model3_cb_sample_weight == 0] = 1
model3_cb_sample_weight.loc[model3_cb_sample_weight == 1] = 1
model3_cb_sample_weight.loc[model3_cb_sample_weight == 2] = 1
model3_cb_sample_weight.loc[model3_cb_sample_weight == 3] = 1
model3_cb_sample_weight.loc[model3_cb_sample_weight == 4] = 1
model3_cb_sample_weight.loc[model3_cb_sample_weight == 5] = 1
for df in [model3_cb_X_train, model3_cb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift ).fillna(0)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift ).fillna(0)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_rolling_forward_center_mean_20'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_std_30'] = df['signal_processed'].rolling(window=30, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_min_25'] = df['signal_processed'].rolling(window=25, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_mean_25_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=25, min_periods=1, center=False ).mean()
df['signal_processed_rolling_forward_min_15_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=15, min_periods=1, center=False ).min()
df['signal_processed_rolling_forward_center_maxminratio_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).max() / df['signal_processed'].rolling(window=15, min_periods=1, center=True ).min()
model3_cb_parameters = {
'n_estimators': 550,
'learning_rate': 0.07,
'depth': 6,
'subsample': 0.7,
'bagging_temperature': 1,
'colsample_bylevel': 0.9,
'l2_leaf_reg': 0.05,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
solo_model3 = MultiModelGBDTEnsemble(lgb_parameters=model3_lgb_parameters,
xgb_parameters=model3_xgb_parameters,
cb_parameters=model3_cb_parameters,
model='Blend')
 solo_model3.train_and_predict(lgb_X_train=model3_lgb_X_train, lgb_y_train=model3_lgb_y_train, lgb_X_test=model3_lgb_X_test,
xgb_X_train=model3_xgb_X_train, xgb_y_train=model3_xgb_y_train, xgb_X_test=model3_xgb_X_test,
cb_X_train=model3_cb_X_train, cb_y_train=model3_cb_y_train, cb_X_test=model3_cb_X_test,
num_boosting_round=400, early_stopping_rounds=20,
lgb_sample_weight=model3_lgb_sample_weight, xgb_sample_weight=model3_xgb_sample_weight, cb_sample_weight=model3_cb_sample_weight,
model=3)
 solo_model3.plot_importance()
del model3_lgb_X_train, model3_lgb_y_train, model3_lgb_X_test, model3_lgb_sample_weight, model3_lgb_parameters
del model3_xgb_X_train, model3_xgb_y_train, model3_xgb_X_test, model3_xgb_sample_weight, model3_xgb_parameters
del model3_cb_X_train, model3_cb_y_train, model3_cb_X_test, model3_cb_sample_weight, model3_cb_parameters
del solo_model3
train_word=full_data[:train.shape[0]]
test_word=full_data[train.shape[0]:]<feature_engineering>
print('
print('------------------------------------------------------')
 model4_lgb_features = ['signal_processed']
model4_lgb_X_train = df_train[df_train['model'] == 4][model4_lgb_features].copy(deep=True)
 model4_lgb_y_train = df_train[df_train['model'] == 4]['open_channels'].copy(deep=True)
 model4_lgb_X_test = df_test[df_test['model'] == 4][model4_lgb_features].copy(deep=True)
 model4_lgb_sample_weight = model4_lgb_y_train.copy(deep=True ).reset_index(drop=True)
 model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 0] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 1] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 2] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 3] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 4] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 5] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 6] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 7] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 8] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 9] = 1
model4_lgb_sample_weight.loc[model4_lgb_sample_weight == 10] = 1
for df in [model4_lgb_X_train, model4_lgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_rolling_forward_center_mean_20'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_min_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_center_min_35_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=35, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_min_140'] = df['signal_processed'].rolling(window=140, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_max_140'] = df['signal_processed'].rolling(window=140, min_periods=1, center=True ).max()
model4_lgb_parameters = {
'num_iterations': 500,
'early_stopping_round': 50,
'num_leaves': 2 ** 7,
'learning_rate': 0.08,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 0.9,
'feature_fraction_bynode': 0.8,
'min_data_in_leaf': 20,
'lambda_l1': 0,
'lambda_l2': 0,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model4_xgb_features = ['signal_processed']
model4_xgb_X_train = df_train[df_train['model'] == 4][model4_xgb_features].copy(deep=True)
 model4_xgb_y_train = df_train[df_train['model'] == 4]['open_channels'].copy(deep=True)
 model4_xgb_X_test = df_test[df_test['model'] == 4][model4_xgb_features].copy(deep=True)
 model4_xgb_sample_weight = model4_xgb_y_train.copy(deep=True ).reset_index(drop=True)
 model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 0] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 1] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 2] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 3] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 4] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 5] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 6] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 7] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 8] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 9] = 1
model4_xgb_sample_weight.loc[model4_xgb_sample_weight == 10] = 1
for df in [model4_xgb_X_train, model4_xgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_squarerooted'] = df['signal_processed'] **(1 / 2)
 df['signal_processed_rolling_forward_center_mean_20'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_std_10'] = df['signal_processed'].rolling(window=10, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_min_5'] = df['signal_processed'].rolling(window=5, min_periods=1, center=True ).min()
df['signal_processed_rolling_forward_center_mean_45_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=45, min_periods=1, center=True ).mean()
model4_xgb_parameters = {
'n_estimators': 2 ** 7,
'learning_rate': 0.08,
'colsample_bytree': 0.9,
'colsample_bylevel': 0.9,
'colsample_bynode': 0.8,
'sumbsample': 0.9,
'max_depth': 6,
'gamma': 0,
'min_child_weight': 1,
'lambda': 1,
'alpha': 0.5,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model4_cb_features = ['signal_processed']
model4_cb_X_train = df_train[df_train['model'] == 4][model4_cb_features].copy(deep=True)
 model4_cb_y_train = df_train[df_train['model'] == 4]['open_channels'].copy(deep=True)
 model4_cb_X_test = df_test[df_test['model'] == 4][model4_cb_features].copy(deep=True)
 model4_cb_sample_weight = model4_cb_y_train.copy(deep=True ).reset_index(drop=True)
 model4_cb_sample_weight.loc[model4_cb_sample_weight == 0] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 1] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 2] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 3] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 4] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 5] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 6] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 7] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 8] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 9] = 1
model4_cb_sample_weight.loc[model4_cb_sample_weight == 10] = 1
for df in [model4_cb_X_train, model4_cb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df['signal_processed'].shift(-shift)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_squarerooted'] = df['signal_processed'] **(1 / 2)
 df['signal_processed_rolling_forward_center_mean_15'] = df['signal_processed'].rolling(window=15, min_periods=1, center=True ).mean()
df['signal_processed_rolling_forward_center_std_55'] = df['signal_processed'].rolling(window=55, min_periods=1, center=True ).std()
df['signal_processed_rolling_forward_center_max_20'] = df['signal_processed'].rolling(window=20, min_periods=1, center=True ).max()
df['signal_processed_rolling_forward_center_median_15_difference'] = df['signal_processed'] - df['signal_processed'].rolling(window=15, min_periods=1, center=True ).median()
model4_cb_parameters = {
'n_estimators': 450,
'learning_rate': 0.065,
'depth': 6,
'subsample': 0.7,
'bagging_temperature': 1,
'colsample_bylevel': 0.9,
'l2_leaf_reg': 0.01,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
solo_model4 = MultiModelGBDTEnsemble(lgb_parameters=model4_lgb_parameters,
xgb_parameters=model4_xgb_parameters,
cb_parameters=model4_cb_parameters,
model='Blend')
 solo_model4.train_and_predict(lgb_X_train=model4_lgb_X_train, lgb_y_train=model4_lgb_y_train, lgb_X_test=model4_lgb_X_test,
xgb_X_train=model4_xgb_X_train, xgb_y_train=model4_xgb_y_train, xgb_X_test=model4_xgb_X_test,
cb_X_train=model4_cb_X_train, cb_y_train=model4_cb_y_train, cb_X_test=model4_cb_X_test,
lgb_sample_weight=model4_lgb_sample_weight, xgb_sample_weight=model4_xgb_sample_weight, cb_sample_weight=model4_cb_sample_weight,
num_boosting_round=500, early_stopping_rounds=20,
model=4)
 solo_model4.plot_importance()
del model4_lgb_X_train, model4_lgb_y_train, model4_lgb_X_test, model4_lgb_sample_weight, model4_lgb_parameters
del model4_xgb_X_train, model4_xgb_y_train, model4_xgb_X_test, model4_xgb_sample_weight, model4_xgb_parameters
del model4_cb_X_train, model4_cb_y_train, model4_cb_X_test, model4_cb_sample_weight, model4_cb_parameters
del solo_model4
pos_text_list=list(corpus_sentiment(train_word,'text','positive')[0])
 pos_target_list=list(corpus_sentiment(train_word,'target','positive')[0])
 neg_text_list=list(corpus_sentiment(train_word,'text','negative')[0])
 neg_target_list=list(corpus_sentiment(train_word,'target','negative')[0])
 neutral_text_list=list(corpus_sentiment(train_word,'text','neutral')[0])
 neutral_target_list=list(corpus_sentiment(train_word,'target','neutral')[0])
 pos_common_words_list=list(set(pos_text_list ).intersection(pos_target_list))
neg_common_words_list=list(set(neg_text_list ).intersection(neg_target_list))
neutral_common_words_list=list(set(neutral_text_list ).intersection(neutral_target_list))<compute_train_metric>
print('
print('---------------------------------------------------------------')
 for model in range(5):
print(f'
print('----------------------------------')
 y_true = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
lgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'lgb_solo_model_oof']
xgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'xgb_solo_model_oof']
cb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'cb_solo_model_oof']
equal_blend_oof =(lgb_oof * 0.34)+(xgb_oof * 0.33)+(cb_oof * 0.33)
 print(f'Solo LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(lgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Solo XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(xgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Solo CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(cb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Solo Model {model} Equal Blend OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(equal_blend_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'
print('---------------------------------------')
 global_y_true = df_train.loc[df_train['is_filtered'] == 0, 'open_channels']
global_lgb_oof = df_train.loc[df_train['is_filtered'] == 0, 'lgb_solo_model_oof']
global_xgb_oof = df_train.loc[df_train['is_filtered'] == 0, 'xgb_solo_model_oof']
global_cb_oof = df_train.loc[df_train['is_filtered'] == 0, 'cb_solo_model_oof']
global_equal_blend_oof =(global_lgb_oof * 0.34)+(global_xgb_oof * 0.33)+(global_cb_oof * 0.33)
 print(f'Solo LGB Model Global OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_lgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Solo XGB Model Global OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_xgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Solo CB Model Global OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_cb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Solo Model Global Equal Blend OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_equal_blend_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 del global_y_true, global_lgb_oof, global_xgb_oof, global_cb_oof, global_equal_blend_oof
results_jaccard=[]
for ind,row in train_word.iterrows() :
sentence1 = row.text
sentence2 = row.target
jaccard_score = jaccard(sentence1,sentence2)
 results_jaccard.append([sentence1,sentence2,jaccard_score] )<merge>
print('
print('-------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model01_lgb_features = ['signal_processed', 'model']
model01_lgb_X_train = df_train[df_train['model'] <= 1][model01_lgb_features].copy(deep=True)
 model01_lgb_y_train = df_train[df_train['model'] <= 1]['open_channels'].copy(deep=True)
 model01_lgb_X_test = df_test[df_train['model'] <= 1][model01_lgb_features].copy(deep=True)
 for df in [model01_lgb_X_train, model01_lgb_X_test]:
for shift in range(1, 3):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_std_15'] = df.groupby('model')['signal_processed'].rolling(window=15, min_periods=1, center=False ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
model01_lgb_parameters = {
'num_iterations': 500,
'early_stopping_round': 50,
'num_leaves': 2 ** 6,
'learning_rate': 0.03,
'bagging_fraction': 0.6,
'bagging_freq': 1,
'feature_fraction': 0.7,
'feature_fraction_bynode': 0.7,
'min_data_in_leaf': 20,
'lambda_l1': 0.6,
'lambda_l2': 0.6,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model01_lgb_scores = []
model01_lgb_oof = np.zeros(model01_lgb_X_train.shape[0])
 model01_lgb_y_pred = pd.DataFrame(np.zeros(( model01_lgb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model01_lgb_importance = pd.DataFrame(np.zeros(( model01_lgb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model01_lgb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model01_lgb_X_train, model01_lgb_X_train['model']), 1):
print(f'
Fold {fold}')
 model01_lgb_trn_data = lgb.Dataset(model01_lgb_X_train.iloc[trn_idx, :], label=model01_lgb_y_train.iloc[trn_idx], categorical_feature=['model'])
 model01_lgb_val_data = lgb.Dataset(model01_lgb_X_train.iloc[val_idx, :], label=model01_lgb_y_train.iloc[val_idx], categorical_feature=['model'])
 model01_lgb_model = lgb.train(model01_lgb_parameters, model01_lgb_trn_data, valid_sets=[model01_lgb_trn_data, model01_lgb_val_data], verbose_eval=50)
 model01_lgb_oof_predictions = model01_lgb_model.predict(model01_lgb_X_train.iloc[val_idx, :], num_iteration=model01_lgb_model.best_iteration)
 model01_lgb_oof[val_idx] = model01_lgb_oof_predictions
df_train.loc[model01_lgb_X_train.iloc[val_idx, :].index, 'lgb_duo_model_oof'] = model01_lgb_oof_predictions
model01_lgb_test_predictions = model01_lgb_model.predict(model01_lgb_X_test, num_iteration=model01_lgb_model.best_iteration)
 model01_lgb_y_pred.iloc[:, fold - 1] = model01_lgb_test_predictions
df_test.loc[model01_lgb_X_test.index, f'fold{fold}_lgb_duo_model_predictions'] = model01_lgb_test_predictions
model01_lgb_importance.iloc[:, fold - 1] = model01_lgb_model.feature_importance(importance_type='gain')
 model01_lgb_score = f1_score(model01_lgb_y_train.iloc[val_idx].values, np.round(np.clip(model01_lgb_oof_predictions, model01_lgb_y_train.min() , model01_lgb_y_train.max())) , average='macro')
 model01_lgb_scores.append(model01_lgb_score)
 print('
Duo LGB Fold {} Macro F1-Score {}
'.format(fold, model01_lgb_score))
print('--------------------')
 print(f'Duo LGB Mean Macro F1-Score {np.mean(model01_lgb_scores):.6} [STD:{np.std(model01_lgb_scores):.6}]')
 for model in [0, 1]:
model_oof = df_train.loc[df_train['model'] == model, 'lgb_duo_model_oof']
model_labels = df_train.loc[df_train['model'] == model, 'open_channels']
print(f'Duo LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model01_lgb_y_train.min() , model01_lgb_y_train.max())) , average="macro"):.6}')
 print(f'Duo LGB Model 0-1 OOF(Rounded)Macro F1-Score {f1_score(model01_lgb_y_train, np.round(np.clip(model01_lgb_oof, model01_lgb_y_train.min() , model01_lgb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 6))
model01_lgb_importance['Mean_Importance'] = model01_lgb_importance.sum(axis=1)/ K
model01_lgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model01_lgb_importance.index, data=model01_lgb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Duo LightGBM Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model01_lgb_X_train, model01_lgb_y_train, model01_lgb_X_test, model01_lgb_parameters
del model01_lgb_scores, model01_lgb_oof, model01_lgb_y_pred, model01_lgb_importance
del model01_lgb_trn_data, model01_lgb_val_data, model01_lgb_model, model01_lgb_oof_predictions, model01_lgb_test_predictions
jaccard = pd.DataFrame(results_jaccard,columns=["text","target","jaccard_score"])
 train_word = train_word.merge(jaccard,how='outer' )<compute_test_metric>
print('
print('-------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model01_xgb_features = ['signal_processed', 'model']
model01_xgb_X_train = df_train[df_train['model'] <= 1][model01_xgb_features].copy(deep=True)
 model01_xgb_y_train = df_train[df_train['model'] <= 1]['open_channels'].copy(deep=True)
 model01_xgb_X_test = df_test[df_train['model'] <= 1][model01_xgb_features].copy(deep=True)
 for df in [model01_xgb_X_train, model01_xgb_X_test]:
for shift in range(1, 3):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_std_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=False ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=False ).max().reset_index(level='model')['signal_processed']
df['signal_processed_pct_change+1'] = df.groupby('model')['signal_processed'].pct_change(1 ).fillna(0)
 model01_xgb_parameters = {
'n_estimators': 2 ** 7,
'learning_rate': 0.07,
'colsample_bytree': 0.8,
'colsample_bylevel': 0.8,
'colsample_bynode': 0.7,
'sumbsample': 0.7,
'max_depth': 6,
'gamma': 0.8,
'min_child_weight': 1,
'lambda': 1,
'alpha': 0.1,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model01_xgb_scores = []
model01_xgb_oof = np.zeros(model01_xgb_X_train.shape[0])
 model01_xgb_y_pred = pd.DataFrame(np.zeros(( model01_xgb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model01_xgb_importance = pd.DataFrame(np.zeros(( model01_xgb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model01_xgb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model01_xgb_X_train, model01_xgb_X_train['model']), 1):
print(f'
Fold {fold}')
 model01_xgb_trn_data = xgb.DMatrix(model01_xgb_X_train.iloc[trn_idx, :], label=model01_xgb_y_train.iloc[trn_idx])
 model01_xgb_val_data = xgb.DMatrix(model01_xgb_X_train.iloc[val_idx, :], label=model01_xgb_y_train.iloc[val_idx])
 model01_xgb_model = xgb.train(model01_xgb_parameters, model01_xgb_trn_data, 500, evals=[(model01_xgb_trn_data, 'train'),(model01_xgb_val_data, 'val')], verbose_eval=50, early_stopping_rounds=25)
 model01_xgb_oof_predictions = model01_xgb_model.predict(xgb.DMatrix(model01_xgb_X_train.iloc[val_idx, :]))
model01_xgb_oof[val_idx] = model01_xgb_oof_predictions
df_train.loc[model01_xgb_X_train.iloc[val_idx, :].index, 'xgb_duo_model_oof'] = model01_xgb_oof_predictions
model01_xgb_test_predictions = model01_xgb_model.predict(xgb.DMatrix(model01_xgb_X_test))
model01_xgb_y_pred.iloc[:, fold - 1] = model01_xgb_test_predictions
df_test.loc[model01_xgb_X_test.index, f'fold{fold}_xgb_duo_model_predictions'] = model01_xgb_test_predictions
model01_xgb_importance.iloc[:, fold - 1] = list(model01_xgb_model.get_score(importance_type='gain' ).values())
 model01_xgb_score = f1_score(model01_xgb_y_train.iloc[val_idx].values, np.round(np.clip(model01_xgb_oof_predictions, model01_xgb_y_train.min() , model01_xgb_y_train.max())) , average='macro')
 model01_xgb_scores.append(model01_xgb_score)
 print('
Duo XGB Fold {} Macro F1-Score {}
'.format(fold, model01_xgb_score))
print('--------------------')
 print(f'Duo XGB Mean Macro F1-Score {np.mean(model01_xgb_scores):.6} [STD:{np.std(model01_xgb_scores):.6}]')
 for model in [0, 1]:
model_oof = df_train.loc[df_train['model'] == model, 'xgb_duo_model_oof']
model_labels = df_train.loc[df_train['model'] == model, 'open_channels']
print(f'Duo XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model01_xgb_y_train.min() , model01_xgb_y_train.max())) , average="macro"):.6}')
 print(f'Duo XGB Model 0-1 OOF(Rounded)Macro F1-Score {f1_score(model01_xgb_y_train, np.round(np.clip(model01_xgb_oof, model01_xgb_y_train.min() , model01_xgb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 6))
model01_xgb_importance['Mean_Importance'] = model01_xgb_importance.sum(axis=1)/ K
model01_xgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model01_xgb_importance.index, data=model01_xgb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Duo XGBoost Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model01_xgb_X_train, model01_xgb_y_train, model01_xgb_X_test, model01_xgb_parameters
del model01_xgb_scores, model01_xgb_oof, model01_xgb_y_pred, model01_xgb_importance
del model01_xgb_trn_data, model01_xgb_val_data, model01_xgb_model, model01_xgb_oof_predictions, model01_xgb_test_predictions
print("Jaccard score equals 1 : {}%".format(round(train_word.loc[train_word['jaccard_score']==1].shape[0]/train_word.shape[0]*100,2)))
 print("Jaccard score less than 0.3 : {}%".format(round(train_word.loc[train_word['jaccard_score']<0.3].shape[0]/train_word.shape[0]*100,2)) )<feature_engineering>
print('
print('-------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model01_cb_features = ['signal_processed', 'model']
model01_cb_X_train = df_train[df_train['model'] <= 1][model01_cb_features].copy(deep=True)
 model01_cb_y_train = df_train[df_train['model'] <= 1]['open_channels'].copy(deep=True)
 model01_cb_X_test = df_test[df_train['model'] <= 1][model01_cb_features].copy(deep=True)
 for df in [model01_cb_X_train, model01_cb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_15'] = df.groupby('model')['signal_processed'].rolling(window=15, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_15'] = df.groupby('model')['signal_processed'].rolling(window=60, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_35_difference'] = df['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=False ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_35_difference'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=False ).max().reset_index(level='model')['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=False ).min().reset_index(level='model')['signal_processed']
model01_cb_parameters = {
'n_estimators': 400,
'learning_rate': 0.08,
'depth': 6,
'subsample': 0.9,
'bagging_temperature': 1,
'colsample_bylevel': 0.9,
'l2_leaf_reg': 0,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
model01_cb_scores = []
model01_cb_oof = np.zeros(model01_cb_X_train.shape[0])
 model01_cb_y_pred = pd.DataFrame(np.zeros(( model01_cb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model01_cb_importance = pd.DataFrame(np.zeros(( model01_cb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model01_cb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model01_cb_X_train, model01_cb_X_train['model']), 1):
print(f'
Fold {fold}')
 model01_cb_trn_data = cb.Pool(model01_cb_X_train.iloc[trn_idx, :], label=model01_cb_y_train.iloc[trn_idx], cat_features=[1])
 model01_cb_val_data = cb.Pool(model01_cb_X_train.iloc[val_idx, :], label=model01_cb_y_train.iloc[val_idx], cat_features=[1])
 model01_cb_model = cb.CatBoostRegressor(**model01_cb_parameters)
 model01_cb_model.fit(model01_cb_trn_data)
 model01_cb_oof_predictions = model01_cb_model.predict(model01_cb_val_data)
 model01_cb_oof[val_idx] = model01_cb_oof_predictions
df_train.loc[model01_cb_X_train.iloc[val_idx, :].index, 'cb_duo_model_oof'] = model01_cb_oof_predictions
model01_cb_test_predictions = model01_cb_model.predict(cb.Pool(model01_cb_X_test, cat_features=[1]))
model01_cb_y_pred.iloc[:, fold - 1] = model01_cb_test_predictions
df_test.loc[model01_cb_X_test.index, f'fold{fold}_cb_duo_model_predictions'] = model01_cb_test_predictions
model01_cb_importance.iloc[:, fold - 1] = model01_cb_model.get_feature_importance()
model01_cb_score = f1_score(model01_cb_y_train.iloc[val_idx].values, np.round(np.clip(model01_cb_oof_predictions, model01_cb_y_train.min() , model01_cb_y_train.max())) , average='macro')
 model01_cb_scores.append(model01_cb_score)
 print('
Duo CB Fold {} Macro F1-Score {}
'.format(fold, model01_cb_score))
print('--------------------')
 print(f'Duo CB Mean Macro F1-Score {np.mean(model01_cb_scores):.6} [STD:{np.std(model01_cb_scores):.6}]')
 for model in [0, 1]:
model_oof = df_train.loc[df_train['model'] == model, 'cb_duo_model_oof']
model_labels = df_train.loc[df_train['model'] == model, 'open_channels']
print(f'Duo CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model01_cb_y_train.min() , model01_cb_y_train.max())) , average="macro"):.6}')
 print(f'Duo CB Model 0-1 OOF(Rounded)Macro F1-Score {f1_score(model01_cb_y_train, np.round(np.clip(model01_cb_oof, model01_cb_y_train.min() , model01_cb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 6))
model01_cb_importance['Mean_Importance'] = model01_cb_importance.sum(axis=1)/ K
model01_cb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model01_cb_importance.index, data=model01_cb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Duo CatBoost Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model01_cb_X_train, model01_cb_y_train, model01_cb_X_test, model01_cb_parameters
del model01_cb_scores, model01_cb_oof, model01_cb_y_pred, model01_cb_importance
del model01_cb_trn_data, model01_cb_val_data, model01_cb_model, model01_cb_oof_predictions, model01_cb_test_predictions
train_word['difference']=abs(train_word['text_tweet_length']-train_word['target_tweet_length'] )<define_variables>
print('
print('--------------------------------------------------------------')
 for model in range(2):
print(f'
print('---------------------------------')
 y_true = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
lgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'lgb_duo_model_oof']
xgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'xgb_duo_model_oof']
cb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'cb_duo_model_oof']
equal_blend_oof =(lgb_oof * 0.34)+(xgb_oof * 0.33)+(cb_oof * 0.33)
 print(f'Duo LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(lgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Duo XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(xgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Duo CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(cb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Duo Model {model} Equal Blend OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(equal_blend_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'
print('--------------------------------------')
 global_y_true = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 1), 'open_channels']
global_lgb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 1), 'lgb_duo_model_oof']
global_xgb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 1), 'xgb_duo_model_oof']
global_cb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 1), 'cb_duo_model_oof']
global_equal_blend_oof =(global_lgb_oof * 0.34)+(global_xgb_oof * 0.33)+(global_cb_oof * 0.33)
 print(f'Duo LGB Model Global(0-1)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_lgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Duo XGB Model Global(0-1)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_xgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Duo CB Model Global(0-1)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_cb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Duo Model Global(0-1)Equal Blend OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_equal_blend_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 del global_y_true, global_lgb_oof, global_xgb_oof, global_cb_oof, global_equal_blend_oof
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}<load_from_csv>
print('
print('----------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model012_lgb_features = ['signal_processed', 'model']
model012_lgb_X_train = df_train[(df_train['model'] <= 2)&(df_train['is_filtered'] == 0)][model012_lgb_features].copy(deep=True)
 model012_lgb_y_train = df_train[(df_train['model'] <= 2)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model012_lgb_X_test = df_test[df_test['model'] <= 2][model012_lgb_features].copy(deep=True)
 for df in [model012_lgb_X_train, model012_lgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_20'] = df.groupby('model')['signal_processed'].rolling(window=20, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_40'] = df.groupby('model')['signal_processed'].rolling(window=40, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_40'] = df.groupby('model')['signal_processed'].rolling(window=40, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_median_35_difference'] = df['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).median().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_50_difference'] = df['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=50, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_maxmin_10_difference'] = df.groupby('model')['signal_processed'].rolling(window=10, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=10, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_squarerooted'] = df['signal_processed'] **(1 / 2)
 model012_lgb_parameters = {
'num_iterations': 300,
'early_stopping_round': 50,
'num_leaves': 2 ** 7,
'learning_rate': 0.045,
'bagging_fraction': 0.7,
'bagging_freq': 1,
'feature_fraction': 0.8,
'feature_fraction_bynode': 0.7,
'min_data_in_leaf': 20,
'lambda_l1': 0,
'lambda_l2': 0.5,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model012_lgb_scores = []
model012_lgb_oof = np.zeros(model012_lgb_X_train.shape[0])
 model012_lgb_y_pred = pd.DataFrame(np.zeros(( model012_lgb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model012_lgb_importance = pd.DataFrame(np.zeros(( model012_lgb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model012_lgb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model012_lgb_X_train, model012_lgb_X_train['model']), 1):
print(f'
Fold {fold}')
 model012_lgb_trn_data = lgb.Dataset(model012_lgb_X_train.iloc[trn_idx, :], label=model012_lgb_y_train.iloc[trn_idx], categorical_feature=['model'])
 model012_lgb_val_data = lgb.Dataset(model012_lgb_X_train.iloc[val_idx, :], label=model012_lgb_y_train.iloc[val_idx], categorical_feature=['model'])
 model012_lgb_model = lgb.train(model012_lgb_parameters, model012_lgb_trn_data, valid_sets=[model012_lgb_trn_data, model012_lgb_val_data], verbose_eval=50)
 model012_lgb_oof_predictions = model012_lgb_model.predict(model012_lgb_X_train.iloc[val_idx, :], num_iteration=model012_lgb_model.best_iteration)
 model012_lgb_oof[val_idx] = model012_lgb_oof_predictions
df_train.loc[model012_lgb_X_train.iloc[val_idx, :].index, 'lgb_trio_model_oof'] = model012_lgb_oof_predictions
model012_lgb_test_predictions = model012_lgb_model.predict(model012_lgb_X_test, num_iteration=model012_lgb_model.best_iteration)
 model012_lgb_y_pred.iloc[:, fold - 1] = model012_lgb_test_predictions
df_test.loc[model012_lgb_X_test.index, f'fold{fold}_lgb_trio_model_predictions'] = model012_lgb_test_predictions
model012_lgb_importance.iloc[:, fold - 1] = model012_lgb_model.feature_importance(importance_type='gain')
 model012_lgb_score = f1_score(model012_lgb_y_train.iloc[val_idx].values, np.round(np.clip(model012_lgb_oof_predictions, model012_lgb_y_train.min() , model012_lgb_y_train.max())) , average='macro')
 model012_lgb_scores.append(model012_lgb_score)
 print('
Trio LGB Fold {} Macro F1-Score {}
'.format(fold, model012_lgb_score))
print('--------------------')
 print(f'Trio LGB Mean Macro F1-Score {np.mean(model012_lgb_scores):.6} [STD:{np.std(model012_lgb_scores):.6}]')
 for model in [0, 1, 2]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'lgb_trio_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Trio LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model012_lgb_y_train.min() , model012_lgb_y_train.max())) , average="macro"):.6}')
 print(f'Trio LGB Model 0-1-2 OOF(Rounded)Macro F1-Score {f1_score(model012_lgb_y_train, np.round(np.clip(model012_lgb_oof, model012_lgb_y_train.min() , model012_lgb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 10))
model012_lgb_importance['Mean_Importance'] = model012_lgb_importance.sum(axis=1)/ K
model012_lgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model012_lgb_importance.index, data=model012_lgb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Trio LightGBM Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model012_lgb_X_train, model012_lgb_y_train, model012_lgb_X_test, model012_lgb_parameters
del model012_lgb_scores, model012_lgb_oof, model012_lgb_y_pred, model012_lgb_importance
del model012_lgb_trn_data, model012_lgb_val_data, model012_lgb_model, model012_lgb_oof_predictions, model012_lgb_test_predictions
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 ct=train.shape[0]
input_ids=np.ones(( ct,MAX_LEN),dtype="int32")
 attention_mask=np.zeros(( ct,MAX_LEN),dtype="int32")
 token_type_ids=np.zeros(( ct,MAX_LEN),dtype="int32")
 start_tokens=np.zeros(( ct,MAX_LEN),dtype="int32")
 end_tokens=np.zeros(( ct,MAX_LEN),dtype="int32" )<string_transform>
print('
print('----------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model012_xgb_features = ['signal_processed', 'model']
model012_xgb_X_train = df_train[(df_train['model'] <= 2)&(df_train['is_filtered'] == 0)][model012_xgb_features].copy(deep=True)
 model012_xgb_y_train = df_train[(df_train['model'] <= 2)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model012_xgb_X_test = df_test[df_test['model'] <= 2][model012_xgb_features].copy(deep=True)
 for df in [model012_xgb_X_train, model012_xgb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_35'] = df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_35'] = df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_55'] = df.groupby('model')['signal_processed'].rolling(window=55, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_maxminratio_15'] = df.groupby('model')['signal_processed'].rolling(window=15, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed'] / df.groupby('model')['signal_processed'].rolling(window=15, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_squared'] = df['signal_processed'] ** 2
model012_xgb_parameters = {
'n_estimators': 2 ** 7,
'learning_rate': 0.095,
'colsample_bytree': 0.9,
'colsample_bylevel': 0.9,
'colsample_bynode': 0.9,
'sumbsample': 0.7,
'max_depth': 7,
'gamma': 0,
'min_child_weight': 1,
'lambda': 1,
'alpha': 0,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model012_xgb_scores = []
model012_xgb_oof = np.zeros(model012_xgb_X_train.shape[0])
 model012_xgb_y_pred = pd.DataFrame(np.zeros(( model012_xgb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model012_xgb_importance = pd.DataFrame(np.zeros(( model012_xgb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model012_xgb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model012_xgb_X_train, model012_xgb_X_train['model']), 1):
print(f'
Fold {fold}')
 model012_xgb_trn_data = xgb.DMatrix(model012_xgb_X_train.iloc[trn_idx, :], label=model012_xgb_y_train.iloc[trn_idx])
 model012_xgb_val_data = xgb.DMatrix(model012_xgb_X_train.iloc[val_idx, :], label=model012_xgb_y_train.iloc[val_idx])
 model012_xgb_model = xgb.train(model012_xgb_parameters, model012_xgb_trn_data, 500, evals=[(model012_xgb_trn_data, 'train'),(model012_xgb_val_data, 'val')], verbose_eval=50, early_stopping_rounds=25)
 model012_xgb_oof_predictions = model012_xgb_model.predict(xgb.DMatrix(model012_xgb_X_train.iloc[val_idx, :]))
model012_xgb_oof[val_idx] = model012_xgb_oof_predictions
df_train.loc[model012_xgb_X_train.iloc[val_idx, :].index, 'xgb_trio_model_oof'] = model012_xgb_oof_predictions
model012_xgb_test_predictions = model012_xgb_model.predict(xgb.DMatrix(model012_xgb_X_test))
model012_xgb_y_pred.iloc[:, fold - 1] = model012_xgb_test_predictions
df_test.loc[model012_xgb_X_test.index, f'fold{fold}_xgb_trio_model_predictions'] = model012_xgb_test_predictions
model012_xgb_importance.iloc[:, fold - 1] = list(model012_xgb_model.get_score(importance_type='gain' ).values())
 model012_xgb_score = f1_score(model012_xgb_y_train.iloc[val_idx].values, np.round(np.clip(model012_xgb_oof_predictions, model012_xgb_y_train.min() , model012_xgb_y_train.max())) , average='macro')
 model012_xgb_scores.append(model012_xgb_score)
 print('
Trio XGB Fold {} Macro F1-Score {}
'.format(fold, model012_xgb_score))
print('--------------------')
 print(f'Trio XGB Mean Macro F1-Score {np.mean(model012_xgb_scores):.6} [STD:{np.std(model012_xgb_scores):.6}]')
 for model in [0, 1, 2]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'xgb_trio_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Trio XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model012_xgb_y_train.min() , model012_xgb_y_train.max())) , average="macro"):.6}')
 print(f'Trio XGB Model 0-1-2 OOF(Rounded)Macro F1-Score {f1_score(model012_xgb_y_train, np.round(np.clip(model012_xgb_oof, model012_xgb_y_train.min() , model012_xgb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 10))
model012_xgb_importance['Mean_Importance'] = model012_xgb_importance.sum(axis=1)/ K
model012_xgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model012_xgb_importance.index, data=model012_xgb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Trio XGBoost Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model012_xgb_X_train, model012_xgb_y_train, model012_xgb_X_test, model012_xgb_parameters
del model012_xgb_scores, model012_xgb_oof, model012_xgb_y_pred, model012_xgb_importance
del model012_xgb_trn_data, model012_xgb_val_data, model012_xgb_model, model012_xgb_oof_predictions, model012_xgb_test_predictions
for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []
idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0:
toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1 
<load_from_csv>
print('
print('----------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model012_cb_features = ['signal_processed', 'model']
model012_cb_X_train = df_train[(df_train['model'] <= 2)&(df_train['is_filtered'] == 0)][model012_cb_features].copy(deep=True)
 model012_cb_y_train = df_train[(df_train['model'] <= 2)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model012_cb_X_test = df_test[df_test['model'] <= 2][model012_cb_features].copy(deep=True)
 for df in [model012_cb_X_train, model012_cb_X_test]:
for shift in range(1, 2):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_20'] = df.groupby('model')['signal_processed'].rolling(window=20, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_35'] = df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_10_difference'] = df['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=10, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_50_difference'] = df['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=50, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_maxmindiff_55'] = df.groupby('model')['signal_processed'].rolling(window=55, min_periods=1, center=False ).max().reset_index(level='model')['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=55, min_periods=1, center=False ).min().reset_index(level='model')['signal_processed']
df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_squarerooted'] = df['signal_processed'] **(1 / 2)
 df['signal_processed_pctchange+1'] = df['signal_processed'].pct_change(1)
 model012_cb_parameters = {
'n_estimators': 500,
'learning_rate': 0.085,
'depth': 7,
'subsample': 0.8,
'bagging_temperature': 1,
'colsample_bylevel': 0.9,
'l2_leaf_reg': 0.6,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
model012_cb_scores = []
model012_cb_oof = np.zeros(model012_cb_X_train.shape[0])
 model012_cb_y_pred = pd.DataFrame(np.zeros(( model012_cb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model012_cb_importance = pd.DataFrame(np.zeros(( model012_cb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model012_cb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model012_cb_X_train, model012_cb_X_train['model']), 1):
print(f'
Fold {fold}')
 model012_cb_trn_data = cb.Pool(model012_cb_X_train.iloc[trn_idx, :], label=model012_cb_y_train.iloc[trn_idx], cat_features=[1])
 model012_cb_val_data = cb.Pool(model012_cb_X_train.iloc[val_idx, :], label=model012_cb_y_train.iloc[val_idx], cat_features=[1])
 model012_cb_model = cb.CatBoostRegressor(**model012_cb_parameters)
 model012_cb_model.fit(model012_cb_trn_data)
 model012_cb_oof_predictions = model012_cb_model.predict(model012_cb_val_data)
 model012_cb_oof[val_idx] = model012_cb_oof_predictions
df_train.loc[model012_cb_X_train.iloc[val_idx, :].index, 'cb_trio_model_oof'] = model012_cb_oof_predictions
model012_cb_test_predictions = model012_cb_model.predict(cb.Pool(model012_cb_X_test, cat_features=[1]))
model012_cb_y_pred.iloc[:, fold - 1] = model012_cb_test_predictions
df_test.loc[model012_cb_X_test.index, f'fold{fold}_cb_trio_model_predictions'] = model012_cb_test_predictions
model012_cb_importance.iloc[:, fold - 1] = model012_cb_model.get_feature_importance()
model012_cb_score = f1_score(model012_cb_y_train.iloc[val_idx].values, np.round(np.clip(model012_cb_oof_predictions, model012_cb_y_train.min() , model012_cb_y_train.max())) , average='macro')
 model012_cb_scores.append(model012_cb_score)
 print('
Trio CB Fold {} Macro F1-Score {}
'.format(fold, model012_cb_score))
print('--------------------')
 print(f'Trio CB Mean Macro F1-Score {np.mean(model012_cb_scores):.6} [STD:{np.std(model012_cb_scores):.6}]')
 for model in [0, 1, 2]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'cb_trio_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Trio CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model012_cb_y_train.min() , model012_cb_y_train.max())) , average="macro"):.6}')
 print(f'Trio CB Model 0-1-2 OOF(Rounded)Macro F1-Score {f1_score(model012_cb_y_train, np.round(np.clip(model012_cb_oof, model012_cb_y_train.min() , model012_cb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 10))
model012_cb_importance['Mean_Importance'] = model012_cb_importance.sum(axis=1)/ K
model012_cb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model012_cb_importance.index, data=model012_cb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Trio CatBoost Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model012_cb_X_train, model012_cb_y_train, model012_cb_X_test, model012_cb_parameters
del model012_cb_scores, model012_cb_oof, model012_cb_y_pred, model012_cb_importance
del model012_cb_trn_data, model012_cb_val_data, model012_cb_model, model012_cb_oof_predictions, model012_cb_test_predictions
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct_test = test.shape[0]
input_ids_t = np.ones(( ct_test,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct_test,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct_test,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1 
<load_pretrained>
print('
print('---------------------------------------------------------------')
 for model in range(3):
print(f'
print('----------------------------------')
 y_true = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
lgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'lgb_trio_model_oof']
xgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'xgb_trio_model_oof']
cb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'cb_trio_model_oof']
equal_blend_oof =(lgb_oof * 0.34)+(xgb_oof * 0.33)+(cb_oof * 0.33)
 print(f'Trio LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(lgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Trio XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(xgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Trio CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(cb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Trio Model {model} Equal Blend OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(equal_blend_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'
print('---------------------------------------')
 global_y_true = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 2), 'open_channels']
global_lgb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 2), 'lgb_trio_model_oof']
global_xgb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 2), 'xgb_trio_model_oof']
global_cb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 2), 'cb_trio_model_oof']
global_equal_blend_oof =(global_lgb_oof * 0.34)+(global_xgb_oof * 0.33)+(global_cb_oof * 0.33)
 print(f'Trio LGB Model Global(0-1-2)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_lgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Trio XGB Model Global(0-1-2)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_xgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Trio CB Model Global(0-1-2)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_cb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Trio Model Global(0-1-2)Equal Blend OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_equal_blend_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 del global_y_true, global_lgb_oof, global_xgb_oof, global_cb_oof, global_equal_blend_oof
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model 
<predict_on_test>
print('
print('------------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model0123_lgb_features = ['signal_processed', 'model']
model0123_lgb_X_train = df_train[(df_train['model'] <= 3)&(df_train['is_filtered'] == 0)][model0123_lgb_features].copy(deep=True)
 model0123_lgb_y_train = df_train[(df_train['model'] <= 3)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model0123_lgb_X_test = df_test[df_test['model'] <= 3][model0123_lgb_features].copy(deep=True)
 for df in [model0123_lgb_X_train, model0123_lgb_X_test]:
for shift in range(1, 5):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_rolling_forward_center_mean_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_50'] = df.groupby('model')['signal_processed'].rolling(window=50, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_60'] = df.groupby('model')['signal_processed'].rolling(window=60, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
model0123_lgb_parameters = {
'num_iterations': 400,
'early_stopping_round': 100,
'num_leaves': 2 ** 7,
'learning_rate': 0.1,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 0.9,
'feature_fraction_bynode': 0.9,
'min_data_in_leaf': 20,
'lambda_l1': 0,
'lambda_l2': 0,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model0123_lgb_scores = []
model0123_lgb_oof = np.zeros(model0123_lgb_X_train.shape[0])
 model0123_lgb_y_pred = pd.DataFrame(np.zeros(( model0123_lgb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model0123_lgb_importance = pd.DataFrame(np.zeros(( model0123_lgb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model0123_lgb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model0123_lgb_X_train, model0123_lgb_X_train['model']), 1):
print(f'
Fold {fold}')
 model0123_lgb_trn_data = lgb.Dataset(model0123_lgb_X_train.iloc[trn_idx, :], label=model0123_lgb_y_train.iloc[trn_idx])
 model0123_lgb_val_data = lgb.Dataset(model0123_lgb_X_train.iloc[val_idx, :], label=model0123_lgb_y_train.iloc[val_idx])
 model0123_lgb_model = lgb.train(model0123_lgb_parameters, model0123_lgb_trn_data, valid_sets=[model0123_lgb_trn_data, model0123_lgb_val_data], verbose_eval=50)
 model0123_lgb_oof_predictions = model0123_lgb_model.predict(model0123_lgb_X_train.iloc[val_idx, :], num_iteration=model0123_lgb_model.best_iteration)
 model0123_lgb_oof[val_idx] = model0123_lgb_oof_predictions
df_train.loc[model0123_lgb_X_train.iloc[val_idx, :].index, 'lgb_quad_model_oof'] = model0123_lgb_oof_predictions
model0123_lgb_test_predictions = model0123_lgb_model.predict(model0123_lgb_X_test, num_iteration=model0123_lgb_model.best_iteration)
 model0123_lgb_y_pred.iloc[:, fold - 1] = model0123_lgb_test_predictions
df_test.loc[model0123_lgb_X_test.index, f'fold{fold}_lgb_quad_model_predictions'] = model0123_lgb_test_predictions
model0123_lgb_importance.iloc[:, fold - 1] = model0123_lgb_model.feature_importance(importance_type='gain')
 model0123_lgb_score = f1_score(model0123_lgb_y_train.iloc[val_idx].values, np.round(np.clip(model0123_lgb_oof_predictions, model0123_lgb_y_train.min() , model0123_lgb_y_train.max())) , average='macro')
 model0123_lgb_scores.append(model0123_lgb_score)
 print('
Quad LGB Fold {} Macro F1-Score {}
'.format(fold, model0123_lgb_score))
print('--------------------')
 print(f'Quad LGB Mean Macro F1-Score {np.mean(model0123_lgb_scores):.6} [STD:{np.std(model0123_lgb_scores):.6}]')
 for model in [0, 1, 2, 3]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'lgb_quad_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Quad LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model0123_lgb_y_train.min() , model0123_lgb_y_train.max())) , average="macro"):.6}')
 print(f'Quad LGB Model 0-1-2-3 OOF(Rounded)Macro F1-Score {f1_score(model0123_lgb_y_train, np.round(np.clip(model0123_lgb_oof, model0123_lgb_y_train.min() , model0123_lgb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 10))
model0123_lgb_importance['Mean_Importance'] = model0123_lgb_importance.sum(axis=1)/ K
model0123_lgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model0123_lgb_importance.index, data=model0123_lgb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Quad LightGBM Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model0123_lgb_X_train, model0123_lgb_y_train, model0123_lgb_X_test, model0123_lgb_parameters
del model0123_lgb_scores, model0123_lgb_oof, model0123_lgb_y_pred, model0123_lgb_importance
del model0123_lgb_trn_data, model0123_lgb_val_data, model0123_lgb_model, model0123_lgb_oof_predictions, model0123_lgb_test_predictions
start_time=dt.now()
n_splits=5
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
DISPLAY=1
for i in range(5):
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/model4/v4-roberta-%i.h5'%i)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/n_splits
preds_end += preds[1]/n_splits
end_time=dt.now()
print(" ")
 print(" ")
 print("Time Taken to run above code :",(end_time-start_time ).total_seconds() /60," minutes" )<categorify>
print('
print('------------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model0123_xgb_features = ['signal_processed', 'model']
model0123_xgb_X_train = df_train[(df_train['model'] <= 3)&(df_train['is_filtered'] == 0)][model0123_xgb_features].copy(deep=True)
 model0123_xgb_y_train = df_train[(df_train['model'] <= 3)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model0123_xgb_X_test = df_test[df_test['model'] <= 3][model0123_xgb_features].copy(deep=True)
 for df in [model0123_xgb_X_train, model0123_xgb_X_test]:
for shift in range(1, 3):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_rolling_forward_center_mean_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_20'] = df.groupby('model')['signal_processed'].rolling(window=20, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_40'] = df.groupby('model')['signal_processed'].rolling(window=40, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
model0123_xgb_parameters = {
'n_estimators': 2 ** 8,
'learning_rate': 0.1,
'colsample_bytree': 0.9,
'colsample_bylevel': 0.9,
'colsample_bynode': 0.9,
'sumbsample': 0.8,
'max_depth': 6,
'gamma': 0,
'min_child_weight': 1,
'lambda': 1,
'alpha': 0,
'objective': 'reg:squarederror',
'seed': XGB_SEED,
'boosting_type': 'gbtree',
'tree_method': 'auto',
'silent': True,
'verbose': 1,
'n_jobs': -1,
}
model0123_xgb_scores = []
model0123_xgb_oof = np.zeros(model0123_xgb_X_train.shape[0])
 model0123_xgb_y_pred = pd.DataFrame(np.zeros(( model0123_xgb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model0123_xgb_importance = pd.DataFrame(np.zeros(( model0123_xgb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model0123_xgb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model0123_xgb_X_train, model0123_xgb_X_train['model']), 1):
print(f'
Fold {fold}')
 model0123_xgb_trn_data = xgb.DMatrix(model0123_xgb_X_train.iloc[trn_idx, :], label=model0123_xgb_y_train.iloc[trn_idx])
 model0123_xgb_val_data = xgb.DMatrix(model0123_xgb_X_train.iloc[val_idx, :], label=model0123_xgb_y_train.iloc[val_idx])
 model0123_xgb_model = xgb.train(model0123_xgb_parameters, model0123_xgb_trn_data, 500, evals=[(model0123_xgb_trn_data, 'train'),(model0123_xgb_val_data, 'val')], verbose_eval=50, early_stopping_rounds=25)
 model0123_xgb_oof_predictions = model0123_xgb_model.predict(xgb.DMatrix(model0123_xgb_X_train.iloc[val_idx, :]))
model0123_xgb_oof[val_idx] = model0123_xgb_oof_predictions
df_train.loc[model0123_xgb_X_train.iloc[val_idx, :].index, 'xgb_quad_model_oof'] = model0123_xgb_oof_predictions
model0123_xgb_test_predictions = model0123_xgb_model.predict(xgb.DMatrix(model0123_xgb_X_test))
model0123_xgb_y_pred.iloc[:, fold - 1] = model0123_xgb_test_predictions
df_test.loc[model0123_xgb_X_test.index, f'fold{fold}_xgb_quad_model_predictions'] = model0123_xgb_test_predictions
model0123_xgb_importance.iloc[:, fold - 1] = list(model0123_xgb_model.get_score(importance_type='gain' ).values())
 model0123_xgb_score = f1_score(model0123_xgb_y_train.iloc[val_idx].values, np.round(np.clip(model0123_xgb_oof_predictions, model0123_xgb_y_train.min() , model0123_xgb_y_train.max())) , average='macro')
 model0123_xgb_scores.append(model0123_xgb_score)
 print('
Quad XGB Fold {} Macro F1-Score {}
'.format(fold, model0123_xgb_score))
print('--------------------')
 print(f'Quad XGB Mean Macro F1-Score {np.mean(model0123_xgb_scores):.6} [STD:{np.std(model0123_xgb_scores):.6}]')
 for model in [0, 1, 2, 3]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'xgb_quad_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Quad XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model0123_xgb_y_train.min() , model0123_xgb_y_train.max())) , average="macro"):.6}')
 print(f'Quad XGB Model 0-1-2-3 OOF(Rounded)Macro F1-Score {f1_score(model0123_xgb_y_train, np.round(np.clip(model0123_xgb_oof, model0123_xgb_y_train.min() , model0123_xgb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 10))
model0123_xgb_importance['Mean_Importance'] = model0123_xgb_importance.sum(axis=1)/ K
model0123_xgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model0123_xgb_importance.index, data=model0123_xgb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Quad XGBoost Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model0123_xgb_X_train, model0123_xgb_y_train, model0123_xgb_X_test, model0123_xgb_parameters
del model0123_xgb_scores, model0123_xgb_oof, model0123_xgb_y_pred, model0123_xgb_importance
del model0123_xgb_trn_data, model0123_xgb_val_data, model0123_xgb_model, model0123_xgb_oof_predictions, model0123_xgb_test_predictions
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )<save_to_csv>
print('
print('------------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model0123_cb_features = ['signal_processed', 'model']
model0123_cb_X_train = df_train[(df_train['model'] <= 3)&(df_train['is_filtered'] == 0)][model0123_cb_features].copy(deep=True)
 model0123_cb_y_train = df_train[(df_train['model'] <= 3)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model0123_cb_X_test = df_test[df_test['model'] <= 3][model0123_cb_features].copy(deep=True)
 for df in [model0123_cb_X_train, model0123_cb_X_test]:
for shift in range(1, 4):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_rolling_forward_center_mean_30'] = df.groupby('model')['signal_processed'].rolling(window=30, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_20'] = df.groupby('model')['signal_processed'].rolling(window=20, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_55'] = df.groupby('model')['signal_processed'].rolling(window=55, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
model0123_cb_parameters = {
'n_estimators': 1000,
'learning_rate': 0.09,
'depth': 7,
'subsample': 0.8,
'bagging_temperature': 1,
'colsample_bylevel': 0.9,
'l2_leaf_reg': 0,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
model0123_cb_scores = []
model0123_cb_oof = np.zeros(model0123_cb_X_train.shape[0])
 model0123_cb_y_pred = pd.DataFrame(np.zeros(( model0123_cb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model0123_cb_importance = pd.DataFrame(np.zeros(( model0123_cb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model0123_cb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model0123_cb_X_train, model0123_cb_X_train['model']), 1):
print(f'
Fold {fold}')
 model0123_cb_trn_data = cb.Pool(model0123_cb_X_train.iloc[trn_idx, :], label=model0123_cb_y_train.iloc[trn_idx], cat_features=[1])
 model0123_cb_val_data = cb.Pool(model0123_cb_X_train.iloc[val_idx, :], label=model0123_cb_y_train.iloc[val_idx], cat_features=[1])
 model0123_cb_model = cb.CatBoostRegressor(**model0123_cb_parameters)
 model0123_cb_model.fit(model0123_cb_trn_data)
 model0123_cb_oof_predictions = model0123_cb_model.predict(model0123_cb_val_data)
 model0123_cb_oof[val_idx] = model0123_cb_oof_predictions
df_train.loc[model0123_cb_X_train.iloc[val_idx, :].index, 'cb_quad_model_oof'] = model0123_cb_oof_predictions
model0123_cb_test_predictions = model0123_cb_model.predict(cb.Pool(model0123_cb_X_test, cat_features=[1]))
model0123_cb_y_pred.iloc[:, fold - 1] = model0123_cb_test_predictions
df_test.loc[model0123_cb_X_test.index, f'fold{fold}_cb_quad_model_predictions'] = model0123_cb_test_predictions
model0123_cb_importance.iloc[:, fold - 1] = model0123_cb_model.get_feature_importance()
model0123_cb_score = f1_score(model0123_cb_y_train.iloc[val_idx].values, np.round(np.clip(model0123_cb_oof_predictions, model0123_cb_y_train.min() , model0123_cb_y_train.max())) , average='macro')
 model0123_cb_scores.append(model0123_cb_score)
 print('
Quad CB Fold {} Macro F1-Score {}
'.format(fold, model0123_cb_score))
print('--------------------')
 print(f'Quad CB Mean Macro F1-Score {np.mean(model0123_cb_scores):.6} [STD:{np.std(model0123_cb_scores):.6}]')
 for model in [0, 1, 2, 3]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'cb_quad_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Quad CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model0123_cb_y_train.min() , model0123_cb_y_train.max())) , average="macro"):.6}')
 print(f'Quad CB Model 0-1-2-3 OOF(Rounded)Macro F1-Score {f1_score(model0123_cb_y_train, np.round(np.clip(model0123_cb_oof, model0123_cb_y_train.min() , model0123_cb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 10))
model0123_cb_importance['Mean_Importance'] = model0123_cb_importance.sum(axis=1)/ K
model0123_cb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model0123_cb_importance.index, data=model0123_cb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Quad CatBoost Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model0123_cb_X_train, model0123_cb_y_train, model0123_cb_X_test, model0123_cb_parameters
del model0123_cb_scores, model0123_cb_oof, model0123_cb_y_pred, model0123_cb_importance
del model0123_cb_trn_data, model0123_cb_val_data, model0123_cb_model, model0123_cb_oof_predictions, model0123_cb_test_predictions
test['selected_text'] = all
submission=test[['textID','selected_text']]
submission.to_csv('submission.csv',index=False)
 submission.head(5 )<import_modules>
print('
print('---------------------------------------------------------------')
 for model in range(4):
print(f'
print('----------------------------------')
 y_true = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
lgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'lgb_quad_model_oof']
xgb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'xgb_quad_model_oof']
cb_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'cb_quad_model_oof']
equal_blend_oof =(lgb_oof * 0.34)+(xgb_oof * 0.33)+(cb_oof * 0.33)
 print(f'Quad LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(lgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Quad XGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(xgb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Quad CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(cb_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'Quad Model {model} Equal Blend OOF(Rounded)Macro F1-Score {f1_score(y_true, np.round(np.clip(equal_blend_oof, y_true.min() , y_true.max())) , average="macro"):.6}')
 print(f'
print('---------------------------------------')
 global_y_true = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 3), 'open_channels']
global_lgb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 3), 'lgb_quad_model_oof']
global_xgb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 3), 'xgb_quad_model_oof']
global_cb_oof = df_train.loc[(df_train['is_filtered'] == 0)&(df_train['model'] <= 3), 'cb_quad_model_oof']
global_equal_blend_oof =(global_lgb_oof * 0.34)+(global_xgb_oof * 0.33)+(global_cb_oof * 0.33)
 print(f'Quad LGB Model Global(0-1-2-3)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_lgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Quad XGB Model Global(0-1-2-3)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_xgb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Quad CB Model Global(0-1-2-3)OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_cb_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 print(f'Quad Model Global(0-1-2-3)Equal Blend OOF(Rounded)Macro F1-Score {f1_score(global_y_true, np.round(np.clip(global_equal_blend_oof, global_y_true.min() , global_y_true.max())) , average="macro"):.6}')
 del global_y_true, global_lgb_oof, global_xgb_oof, global_cb_oof, global_equal_blend_oof
import os
import gc
import torch
import pandas as pd
import torch.nn as nn
import numpy as np
import torch.nn.functional as F
from torch.optim import lr_scheduler
from sklearn import model_selection
from sklearn import metrics
import transformers
import tokenizers
from transformers import AdamW, get_linear_schedule_with_warmup
from tqdm.autonotebook import tqdm
import utils<install_modules>
print('
print('---------------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model01234_lgb_features = ['signal_processed', 'model']
model01234_lgb_X_train = df_train[df_train['is_filtered'] == 0][model01234_lgb_features].copy(deep=True)
 model01234_lgb_y_train = df_train[df_train['is_filtered'] == 0]['open_channels'].copy(deep=True)
 model01234_lgb_X_test = df_test[df_train['is_filtered'] == 0][model01234_lgb_features].copy(deep=True)
 for df in [model01234_lgb_X_train, model01234_lgb_X_test]:
for shift in range(1, 6):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 for diff in range(1, 3):
df[f'signal_processed_diff+{diff}'] = df.groupby('model')['signal_processed'].diff(diff)
 df[f'signal_processed_diff-{diff}'] = df.groupby('model')['signal_processed'].diff(-diff)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_squarerooted'] = df['signal_processed'] **(1 / 2)
 df['signal_processed_abs'] = np.abs(df['signal_processed'])
 df['signal_processed_fraction'] = np.abs(df['signal_processed'] - df['signal_processed'].astype(int))
df['signal_processed_rolling_forward_center_mean_50'] = df.groupby('model')['signal_processed'].rolling(window=50, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_35'] = df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_35'] = df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_var_60'] = df.groupby('model')['signal_processed'].rolling(window=60, min_periods=1, center=True ).var().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_mean_40_difference'] = df['signal_processed'] - df.groupby('model')['signal_processed'].rolling(window=40, min_periods=1, center=False ).mean().reset_index(level='model')['signal_processed']
model01234_lgb_parameters = {
'num_iterations': 1500,
'early_stopping_round': 150,
'num_leaves': 2 ** 7,
'learning_rate': 0.025,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 0.8,
'feature_fraction_bynode': 0.6,
'min_data_in_leaf': 20,
'lambda_l1': 0,
'lambda_l2': 0,
'max_depth': -1,
'objective': 'regression',
'seed': LGB_SEED,
'feature_fraction_seed': LGB_SEED,
'bagging_seed': LGB_SEED,
'drop_seed': LGB_SEED,
'data_random_seed': LGB_SEED,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'rmse',
'n_jobs': -1,
}
model01234_lgb_scores = []
model01234_lgb_oof = np.zeros(model01234_lgb_X_train.shape[0])
 model01234_lgb_y_pred = pd.DataFrame(np.zeros(( model01234_lgb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model01234_lgb_importance = pd.DataFrame(np.zeros(( model01234_lgb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model01234_lgb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model01234_lgb_X_train, model01234_lgb_X_train['model']), 1):
print(f'
Fold {fold}')
 model01234_lgb_trn_data = lgb.Dataset(model01234_lgb_X_train.iloc[trn_idx, :], label=model01234_lgb_y_train.iloc[trn_idx], categorical_feature=['model'])
 model01234_lgb_val_data = lgb.Dataset(model01234_lgb_X_train.iloc[val_idx, :], label=model01234_lgb_y_train.iloc[val_idx], categorical_feature=['model'])
 model01234_lgb_model = lgb.train(model01234_lgb_parameters, model01234_lgb_trn_data, valid_sets=[model01234_lgb_trn_data, model01234_lgb_val_data], verbose_eval=50)
 model01234_lgb_oof_predictions = model01234_lgb_model.predict(model01234_lgb_X_train.iloc[val_idx, :], num_iteration=model01234_lgb_model.best_iteration)
 model01234_lgb_oof[val_idx] = model01234_lgb_oof_predictions
df_train.loc[model01234_lgb_X_train.iloc[val_idx, :].index, 'lgb_penta_model_oof'] = model01234_lgb_oof_predictions
model01234_lgb_test_predictions = model01234_lgb_model.predict(model01234_lgb_X_test, num_iteration=model01234_lgb_model.best_iteration)
 model01234_lgb_y_pred.iloc[:, fold - 1] = model01234_lgb_test_predictions
df_test.loc[model01234_lgb_X_test.index, f'fold{fold}_lgb_penta_model_predictions'] = model01234_lgb_test_predictions
model01234_lgb_importance.iloc[:, fold - 1] = model01234_lgb_model.feature_importance(importance_type='gain')
 model01234_lgb_score = f1_score(model01234_lgb_y_train.iloc[val_idx].values, np.round(np.clip(model01234_lgb_oof_predictions, model01234_lgb_y_train.min() , model01234_lgb_y_train.max())) , average='macro')
 model01234_lgb_scores.append(model01234_lgb_score)
 print('
Penta LGB Fold {} Macro F1-Score {}
'.format(fold, model01234_lgb_score))
print('--------------------')
 print(f'Penta LGB Mean Macro F1-Score {np.mean(model01234_lgb_scores):.6} [STD:{np.std(model01234_lgb_scores):.6}]')
 for model in [0, 1, 2, 3, 4]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'lgb_penta_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Penta LGB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model01234_lgb_y_train.min() , model01234_lgb_y_train.max())) , average="macro"):.6}')
 print(f'Penta LGB Model 0-1-2-3-4 OOF(Rounded)Macro F1-Score {f1_score(model01234_lgb_y_train, np.round(np.clip(model01234_lgb_oof, model01234_lgb_y_train.min() , model01234_lgb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 16))
model01234_lgb_importance['Mean_Importance'] = model01234_lgb_importance.sum(axis=1)/ K
model01234_lgb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model01234_lgb_importance.index, data=model01234_lgb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Penta LightGBM Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model01234_lgb_X_train, model01234_lgb_y_train, model01234_lgb_X_test, model01234_lgb_parameters
del model01234_lgb_scores, model01234_lgb_oof, model01234_lgb_y_pred, model01234_lgb_importance
del model01234_lgb_trn_data, model01234_lgb_val_data, model01234_lgb_model, model01234_lgb_oof_predictions, model01234_lgb_test_predictions
apex_on = True
if apex_on:
! pip install --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext".. /input/nvidiaapex/.
 if apex_on: 
<define_variables>
print('
print('---------------------------------------------------------------')
 K = 2
skf = StratifiedKFold(n_splits=K)
 print(f'
model01234_cb_features = ['signal_processed', 'model']
model01234_cb_X_train = df_train[df_train['is_filtered'] == 0][model01234_cb_features].copy(deep=True)
 model01234_cb_y_train = df_train[df_train['is_filtered'] == 0]['open_channels'].copy(deep=True)
 model01234_cb_X_test = df_test[df_train['is_filtered'] == 0][model01234_cb_features].copy(deep=True)
 for df in [model01234_cb_X_train, model01234_cb_X_test]:
for shift in range(1, 6):
df[f'signal_processed_shift+{shift}'] = df.groupby('model')['signal_processed'].shift(shift)
 df[f'signal_processed_shift-{shift}'] = df.groupby('model')['signal_processed'].shift(-shift)
 df['signal_processed_squared'] = df['signal_processed'] ** 2
df['signal_processed_squarerooted'] = df['signal_processed'] **(1 / 2)
 df['signal_processed_fraction'] = np.abs(df['signal_processed'] - df['signal_processed'].astype(int))
df['signal_processed_rolling_forward_center_mean_50'] = df.groupby('model')['signal_processed'].rolling(window=50, min_periods=1, center=True ).mean().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_std_25'] = df.groupby('model')['signal_processed'].rolling(window=25, min_periods=1, center=True ).std().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_min_35'] = df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).min().reset_index(level='model')['signal_processed']
df['signal_processed_rolling_forward_center_max_35'] = df.groupby('model')['signal_processed'].rolling(window=35, min_periods=1, center=True ).max().reset_index(level='model')['signal_processed']
model01234_cb_parameters = {
'n_estimators': 1000,
'learning_rate': 0.075,
'depth': 7,
'subsample': 0.6,
'bagging_temperature': 1,
'colsample_bylevel': 0.9,
'l2_leaf_reg': 0.1,
'metric_period': 50,
'boost_from_average': True,
'eval_metric': 'RMSE',
'loss_function': 'RMSE',
'random_seed': CB_SEED,
'verbose': 1,
}
model01234_cb_scores = []
model01234_cb_oof = np.zeros(model01234_cb_X_train.shape[0])
 model01234_cb_y_pred = pd.DataFrame(np.zeros(( model01234_cb_X_test.shape[0], K)) , columns=[f'Fold_{i}_Predictions' for i in range(1, K + 1)])
 model01234_cb_importance = pd.DataFrame(np.zeros(( model01234_cb_X_train.shape[1], K)) , columns=[f'Fold_{i}_Importance' for i in range(1, K + 1)], index=model01234_cb_X_train.columns)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(model01234_cb_X_train, model01234_cb_X_train['model']), 1):
print(f'
Fold {fold}')
 model01234_cb_trn_data = cb.Pool(model01234_cb_X_train.iloc[trn_idx, :], label=model01234_cb_y_train.iloc[trn_idx], cat_features=[1])
 model01234_cb_val_data = cb.Pool(model01234_cb_X_train.iloc[val_idx, :], label=model01234_cb_y_train.iloc[val_idx], cat_features=[1])
 model01234_cb_model = cb.CatBoostRegressor(**model01234_cb_parameters)
 model01234_cb_model.fit(model01234_cb_trn_data)
 model01234_cb_oof_predictions = model01234_cb_model.predict(model01234_cb_val_data)
 model01234_cb_oof[val_idx] = model01234_cb_oof_predictions
df_train.loc[model01234_cb_X_train.iloc[val_idx, :].index, 'cb_penta_model_oof'] = model01234_cb_oof_predictions
model01234_cb_test_predictions = model01234_cb_model.predict(cb.Pool(model01234_cb_X_test, cat_features=[1]))
model01234_cb_y_pred.iloc[:, fold - 1] = model01234_cb_test_predictions
df_test.loc[model01234_cb_X_test.index, f'fold{fold}_cb_penta_model_predictions'] = model01234_cb_test_predictions
model01234_cb_importance.iloc[:, fold - 1] = model01234_cb_model.get_feature_importance()
model01234_cb_score = f1_score(model01234_cb_y_train.iloc[val_idx].values, np.round(np.clip(model01234_cb_oof_predictions, model01234_cb_y_train.min() , model01234_cb_y_train.max())) , average='macro')
 model01234_cb_scores.append(model01234_cb_score)
 print('
Penta CB Fold {} Macro F1-Score {}
'.format(fold, model01234_cb_score))
print('--------------------')
 print(f'Penta CB Mean Macro F1-Score {np.mean(model01234_cb_scores):.6} [STD:{np.std(model01234_cb_scores):.6}]')
 for model in [0, 1, 2, 3, 4]:
model_oof = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'cb_penta_model_oof']
model_labels = df_train.loc[(df_train['model'] == model)&(df_train['is_filtered'] == 0), 'open_channels']
print(f'Penta CB Model {model} OOF(Rounded)Macro F1-Score {f1_score(model_labels, np.round(np.clip(model_oof, model01234_cb_y_train.min() , model01234_cb_y_train.max())) , average="macro"):.6}')
 print(f'Penta CB Model 0-1-2-3-4 OOF(Rounded)Macro F1-Score {f1_score(model01234_cb_y_train, np.round(np.clip(model01234_cb_oof, model01234_cb_y_train.min() , model01234_cb_y_train.max())) , average="macro"):.6}')
 print('--------------------')
 plt.figure(figsize=(20, 16))
model01234_cb_importance['Mean_Importance'] = model01234_cb_importance.sum(axis=1)/ K
model01234_cb_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=model01234_cb_importance.index, data=model01234_cb_importance)
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('Penta CatBoost Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del model01234_cb_X_train, model01234_cb_y_train, model01234_cb_X_test, model01234_cb_parameters
del model01234_cb_scores, model01234_cb_oof, model01234_cb_y_pred, model01234_cb_importance
del model01234_cb_trn_data, model01234_cb_val_data, model01234_cb_model, model01234_cb_oof_predictions, model01234_cb_test_predictions
cdf_threshold =.5<define_variables>
print('---------- Model 0 ----------
')
 model0_labels = df_train[df_train['model'] == 0]['open_channels'].copy(deep=True)
 model0_solo_lgb_oof_score = f1_score(model0_labels, np.round(df_train[df_train['model'] == 0]['lgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model0_solo_xgb_oof_score = f1_score(model0_labels, np.round(df_train[df_train['model'] == 0]['xgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model0_solo_cb_oof_score = f1_score(model0_labels, np.round(df_train[df_train['model'] == 0]['cb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model0_duo_lgb_oof_score = f1_score(model0_labels, np.round(df_train[df_train['model'] == 0]['lgb_duo_model_oof'] ).astype(np.uint8), average='macro')
 model0_duo_xgb_oof_score = f1_score(model0_labels, np.round(df_train[df_train['model'] == 0]['xgb_duo_model_oof'] ).astype(np.uint8), average='macro')
 model0_duo_cb_oof_score = f1_score(model0_labels, np.round(df_train[df_train['model'] == 0]['cb_duo_model_oof'] ).astype(np.uint8), average='macro')
 df_train.loc[df_train['model'] == 0, 'blend_oof'] =(df_train.loc[df_train['model'] == 0, 'lgb_solo_model_oof'] * 0.17)+\ 
(df_train.loc[df_train['model'] == 0, 'xgb_solo_model_oof'] * 0.165)+\ 
(df_train.loc[df_train['model'] == 0, 'cb_solo_model_oof'] * 0.165)+\ 
(df_train.loc[df_train['model'] == 0, 'lgb_duo_model_oof'] * 0.17)+\ 
(df_train.loc[df_train['model'] == 0, 'xgb_duo_model_oof'] * 0.165)+\ 
(df_train.loc[df_train['model'] == 0, 'cb_duo_model_oof'] * 0.165)
 model0_blend_oof_score = f1_score(model0_labels, np.round(df_train[df_train['model'] == 0]['blend_oof'] ).astype(np.uint8), average='macro')
 print(f'Solo LGB Model 0 OOF(Rounded)Macro F1-Score {model0_solo_lgb_oof_score:.6}')
 print(f'Solo XGB OOF Model 0(Rounded)Macro F1-Score {model0_solo_xgb_oof_score:.6}')
 print(f'Solo CB OOF Model 0(Rounded)Macro F1-Score {model0_solo_cb_oof_score:.6}')
 print(f'Duo LGB Model 0 OOF(Rounded)Macro F1-Score {model0_duo_lgb_oof_score:.6}')
 print(f'Duo XGB OOF Model 0(Rounded)Macro F1-Score {model0_duo_xgb_oof_score:.6}')
 print(f'Duo CB OOF Model 0(Rounded)Macro F1-Score {model0_duo_cb_oof_score:.6}')
 print(f'Blend OOF Model 0(Rounded)Macro F1-Score {model0_blend_oof_score:.6}')
 print('
---------- Model 1 ----------
')
 model1_labels = df_train[df_train['model'] == 1]['open_channels'].copy(deep=True)
 model1_solo_lgb_oof_score = f1_score(model1_labels, np.round(df_train[df_train['model'] == 1]['lgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model1_solo_xgb_oof_score = f1_score(model1_labels, np.round(df_train[df_train['model'] == 1]['xgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model1_solo_cb_oof_score = f1_score(model1_labels, np.round(df_train[df_train['model'] == 1]['cb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model1_duo_lgb_oof_score = f1_score(model1_labels, np.round(df_train[df_train['model'] == 1]['lgb_duo_model_oof'] ).astype(np.uint8), average='macro')
 model1_duo_xgb_oof_score = f1_score(model1_labels, np.round(df_train[df_train['model'] == 1]['xgb_duo_model_oof'] ).astype(np.uint8), average='macro')
 model1_duo_cb_oof_score = f1_score(model1_labels, np.round(df_train[df_train['model'] == 1]['cb_duo_model_oof'] ).astype(np.uint8), average='macro')
 df_train.loc[df_train['model'] == 1, 'blend_oof'] =(df_train.loc[df_train['model'] == 1, 'lgb_solo_model_oof'] * 0.17)+\ 
(df_train.loc[df_train['model'] == 1, 'xgb_solo_model_oof'] * 0.165)+\ 
(df_train.loc[df_train['model'] == 1, 'cb_solo_model_oof'] * 0.165)+\ 
(df_train.loc[df_train['model'] == 1, 'lgb_duo_model_oof'] * 0.17)+\ 
(df_train.loc[df_train['model'] == 1, 'xgb_duo_model_oof'] * 0.165)+\ 
(df_train.loc[df_train['model'] == 1, 'cb_duo_model_oof'] * 0.165)
 model1_blend_oof_score = f1_score(model1_labels, np.round(df_train[df_train['model'] == 1]['blend_oof'] ).astype(np.uint8), average='macro')
 print(f'Solo LGB Model 1 OOF(Rounded)Macro F1-Score {model1_solo_lgb_oof_score:.6}')
 print(f'Solo XGB OOF Model 1(Rounded)Macro F1-Score {model1_solo_xgb_oof_score:.6}')
 print(f'Solo CB OOF Model 1(Rounded)Macro F1-Score {model1_solo_cb_oof_score:.6}')
 print(f'Duo LGB Model 1 OOF(Rounded)Macro F1-Score {model1_duo_lgb_oof_score:.6}')
 print(f'Duo XGB OOF Model 1(Rounded)Macro F1-Score {model1_duo_xgb_oof_score:.6}')
 print(f'Duo CB OOF Model 1(Rounded)Macro F1-Score {model1_duo_cb_oof_score:.6}')
 print(f'Blend OOF Model 1(Rounded)Macro F1-Score {model1_blend_oof_score:.6}')
 print('
---------- Model 2 ----------
')
 model2_labels = df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['open_channels'].copy(deep=True)
 model2_solo_lgb_oof_score = f1_score(model2_labels, np.round(df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['lgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model2_solo_xgb_oof_score = f1_score(model2_labels, np.round(df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['xgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model2_solo_cb_oof_score = f1_score(model2_labels, np.round(df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['cb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model2_trio_lgb_oof_score = f1_score(model2_labels, np.round(df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['lgb_trio_model_oof'] ).astype(np.uint8), average='macro')
 model2_trio_xgb_oof_score = f1_score(model2_labels, np.round(df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['xgb_trio_model_oof'] ).astype(np.uint8), average='macro')
 model2_trio_cb_oof_score = f1_score(model2_labels, np.round(df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['cb_trio_model_oof'] ).astype(np.uint8), average='macro')
 df_train.loc[(df_train['model'] == 2)&(df_train['is_filtered'] == 0), 'blend_oof'] =(df_train.loc[(df_train['model'] == 2)&(df_train['is_filtered'] == 0), 'lgb_solo_model_oof'] * 0.14)+\ 
(df_train.loc[(df_train['model'] == 2)&(df_train['is_filtered'] == 0), 'xgb_solo_model_oof'] * 0.13)+\ 
(df_train.loc[(df_train['model'] == 2)&(df_train['is_filtered'] == 0), 'cb_solo_model_oof'] * 0.13)+\ 
(df_train.loc[(df_train['model'] == 2)&(df_train['is_filtered'] == 0), 'lgb_trio_model_oof'] * 0.2)+\ 
(df_train.loc[(df_train['model'] == 2)&(df_train['is_filtered'] == 0), 'xgb_trio_model_oof'] * 0.2)+\ 
(df_train.loc[(df_train['model'] == 2)&(df_train['is_filtered'] == 0), 'cb_trio_model_oof'] * 0.2)
 model2_blend_oof_score = f1_score(model2_labels, np.round(df_train[(df_train['model'] == 2)&(df_train['is_filtered'] == 0)]['blend_oof'] ).astype(np.uint8), average='macro')
 print(f'Solo LGB Model 2 OOF(Rounded)Macro F1-Score {model2_solo_lgb_oof_score:.6}')
 print(f'Solo XGB OOF Model 2(Rounded)Macro F1-Score {model2_solo_xgb_oof_score:.6}')
 print(f'Solo CB OOF Model 2(Rounded)Macro F1-Score {model2_solo_cb_oof_score:.6}')
 print(f'Trio LGB Model 2 OOF(Rounded)Macro F1-Score {model2_trio_lgb_oof_score:.6}')
 print(f'Trio XGB OOF Model 2(Rounded)Macro F1-Score {model2_trio_xgb_oof_score:.6}')
 print(f'Trio CB OOF Model 2(Rounded)Macro F1-Score {model2_trio_cb_oof_score:.6}')
 print(f'Blend OOF Model 2(Rounded)Macro F1-Score {model2_blend_oof_score:.6}')
 print('
---------- Model 3 ----------
')
 model3_labels = df_train[df_train['model'] == 3]['open_channels'].copy(deep=True)
 model3_solo_lgb_oof_score = f1_score(model3_labels, np.round(df_train[df_train['model'] == 3]['lgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model3_solo_xgb_oof_score = f1_score(model3_labels, np.round(df_train[df_train['model'] == 3]['xgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model3_solo_cb_oof_score = f1_score(model3_labels, np.round(df_train[df_train['model'] == 3]['cb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model3_quad_lgb_oof_score = f1_score(model3_labels, np.round(df_train[df_train['model'] == 3]['lgb_quad_model_oof'] ).astype(np.uint8), average='macro')
 model3_quad_xgb_oof_score = f1_score(model3_labels, np.round(df_train[df_train['model'] == 3]['xgb_quad_model_oof'] ).astype(np.uint8), average='macro')
 model3_quad_cb_oof_score = f1_score(model3_labels, np.round(df_train[df_train['model'] == 3]['cb_quad_model_oof'] ).astype(np.uint8), average='macro')
 df_train.loc[df_train['model'] == 3, 'blend_oof'] =(df_train.loc[df_train['model'] == 3, 'lgb_solo_model_oof'] * 0.14)+\ 
(df_train.loc[df_train['model'] == 3, 'xgb_solo_model_oof'] * 0.13)+\ 
(df_train.loc[df_train['model'] == 3, 'cb_solo_model_oof'] * 0.13)+\ 
(df_train.loc[df_train['model'] == 3, 'lgb_quad_model_oof'] * 0.2)+\ 
(df_train.loc[df_train['model'] == 3, 'xgb_quad_model_oof'] * 0.2)+\ 
(df_train.loc[df_train['model'] == 3, 'cb_quad_model_oof'] * 0.2)
 model3_blend_oof_score = f1_score(model3_labels, np.round(df_train[df_train['model'] == 3]['blend_oof'] ).astype(np.uint8), average='macro')
 print(f'Solo LGB Model 3 OOF(Rounded)Macro F1-Score {model3_solo_lgb_oof_score:.6}')
 print(f'Solo XGB OOF Model 3(Rounded)Macro F1-Score {model3_solo_xgb_oof_score:.6}')
 print(f'Solo CB OOF Model 3(Rounded)Macro F1-Score {model3_solo_cb_oof_score:.6}')
 print(f'Quad LGB Model 3 OOF(Rounded)Macro F1-Score {model3_quad_lgb_oof_score:.6}')
 print(f'Quad XGB OOF Model 3(Rounded)Macro F1-Score {model3_quad_xgb_oof_score:.6}')
 print(f'Quad CB OOF Model 3(Rounded)Macro F1-Score {model3_quad_cb_oof_score:.6}')
 print(f'Blend OOF Model 3(Rounded)Macro F1-Score {model3_blend_oof_score:.6}')
 print('
---------- Model 4 ----------
')
 model4_labels = df_train[df_train['model'] == 4]['open_channels'].copy(deep=True)
 model4_solo_lgb_oof_score = f1_score(model4_labels, np.round(df_train[df_train['model'] == 4]['lgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model4_solo_xgb_oof_score = f1_score(model4_labels, np.round(df_train[df_train['model'] == 4]['xgb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model4_solo_cb_oof_score = f1_score(model4_labels, np.round(df_train[df_train['model'] == 4]['cb_solo_model_oof'] ).astype(np.uint8), average='macro')
 model4_penta_lgb_oof_score = f1_score(model4_labels, np.round(df_train[df_train['model'] == 4]['lgb_penta_model_oof'] ).astype(np.uint8), average='macro')
 model4_penta_cb_oof_score = f1_score(model4_labels, np.round(df_train[df_train['model'] == 4]['cb_penta_model_oof'] ).astype(np.uint8), average='macro')
 df_train.loc[df_train['model'] == 4, 'blend_oof'] =(df_train.loc[df_train['model'] == 4, 'lgb_solo_model_oof'] * 0.034)+\ 
(df_train.loc[df_train['model'] == 4, 'xgb_solo_model_oof'] * 0.033)+\ 
(df_train.loc[df_train['model'] == 4, 'cb_solo_model_oof'] * 0.033)+\ 
(df_train.loc[df_train['model'] == 4, 'lgb_penta_model_oof'] * 0.45)+\ 
(df_train.loc[df_train['model'] == 4, 'cb_penta_model_oof'] * 0.45)
 model4_blend_oof_score = f1_score(model4_labels, np.round(df_train[df_train['model'] == 4]['blend_oof'] ).astype(np.uint8), average='macro')
 print(f'Solo LGB Model 4 OOF(Rounded)Macro F1-Score {model4_solo_lgb_oof_score:.6}')
 print(f'Solo XGB OOF Model 4(Rounded)Macro F1-Score {model4_solo_xgb_oof_score:.6}')
 print(f'Solo CB OOF Model 4(Rounded)Macro F1-Score {model4_solo_cb_oof_score:.6}')
 print(f'Penta LGB Model 4 OOF(Rounded)Macro F1-Score {model4_penta_lgb_oof_score:.6}')
 print(f'Penta CB OOF Model 4(Rounded)Macro F1-Score {model4_penta_cb_oof_score:.6}')
 print(f'Blend OOF Model 4(Rounded)Macro F1-Score {model4_blend_oof_score:.6}')
 print('
---------- Global ----------
')
 global_blend_oof_score = f1_score(df_train[df_train['is_filtered'] == 0]['open_channels'], np.round(df_train[df_train['is_filtered'] == 0]['blend_oof'] ).astype(np.uint8), average='macro')
 print(f'Global Blend OOF(Rounded)Macro F1-Score {global_blend_oof_score:.6}
' )
class config:
TRAIN_BATCH_SIZE = 48
VALID_BATCH_SIZE = 8
if apex_on:
TRAIN_BATCH_SIZE *= 2
VALID_BATCH_SIZE *= 2
EPOCHS = 4
LR = 1e-4
ROBERTA_PATH = ".. /input/roberta-base/"
MODEL_PATH = "model.pth"
TRAINING_FILE = ".. /input/tweet-train-folds/train_folds.csv"
TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f".. /input/roberta-base/vocab.json",
merges_file=f".. /input/roberta-base/merges.txt",
lowercase=True,
add_prefix_space=True
)<string_transform>
df_test['lgb_solo_model_predictions'] =(df_test['fold1_lgb_solo_model_predictions'] + df_test['fold2_lgb_solo_model_predictions'])/ 2
df_test['xgb_solo_model_predictions'] =(df_test['fold1_xgb_solo_model_predictions'] + df_test['fold2_xgb_solo_model_predictions'])/ 2
df_test['cb_solo_model_predictions'] =(df_test['fold1_cb_solo_model_predictions'] + df_test['fold2_cb_solo_model_predictions'])/ 2
df_test['lgb_duo_model_predictions'] =(df_test['fold1_lgb_duo_model_predictions'] + df_test['fold2_lgb_duo_model_predictions'])/ 2
df_test['xgb_duo_model_predictions'] =(df_test['fold1_xgb_duo_model_predictions'] + df_test['fold2_xgb_duo_model_predictions'])/ 2
df_test['cb_duo_model_predictions'] =(df_test['fold1_cb_duo_model_predictions'] + df_test['fold2_cb_duo_model_predictions'])/ 2
df_test['lgb_trio_model_predictions'] =(df_test['fold1_lgb_trio_model_predictions'] + df_test['fold2_lgb_trio_model_predictions'])/ 2
df_test['xgb_trio_model_predictions'] =(df_test['fold1_xgb_trio_model_predictions'] + df_test['fold2_xgb_trio_model_predictions'])/ 2
df_test['cb_trio_model_predictions'] =(df_test['fold1_cb_trio_model_predictions'] + df_test['fold2_cb_trio_model_predictions'])/ 2
df_test['lgb_quad_model_predictions'] =(df_test['fold1_lgb_quad_model_predictions'] + df_test['fold2_lgb_quad_model_predictions'])/ 2
df_test['xgb_quad_model_predictions'] =(df_test['fold1_xgb_quad_model_predictions'] + df_test['fold2_xgb_quad_model_predictions'])/ 2
df_test['cb_quad_model_predictions'] =(df_test['fold1_cb_quad_model_predictions'] + df_test['fold2_cb_quad_model_predictions'])/ 2
df_test['lgb_penta_model_predictions'] =(df_test['fold1_lgb_penta_model_predictions'] + df_test['fold2_lgb_penta_model_predictions'])/ 2
df_test['cb_penta_model_predictions'] =(df_test['fold1_cb_penta_model_predictions'] + df_test['fold2_cb_penta_model_predictions'])/ 2
for fold in range(1, 3):
for gbdt in ['lgb', 'xgb', 'cb']:
for model_type in ['solo', 'duo', 'trio', 'quad']:
df_test.drop(columns=[f'fold{fold}_{gbdt}_{model_type}_model_predictions'], inplace=True)
 df_test.drop(columns=['fold1_lgb_penta_model_predictions', 'fold2_lgb_penta_model_predictions', 'fold1_cb_penta_model_predictions', 'fold2_cb_penta_model_predictions'], inplace=True)
 df_test.loc[df_test['model'] == 0, 'blend_predictions'] =(df_test.loc[df_test['model'] == 0, 'lgb_solo_model_predictions'] * 0.17)+\ 
(df_test.loc[df_test['model'] == 0, 'xgb_solo_model_predictions'] * 0.165)+\ 
(df_test.loc[df_test['model'] == 0, 'cb_solo_model_predictions'] * 0.165)+\ 
(df_test.loc[df_test['model'] == 0, 'lgb_duo_model_predictions'] * 0.17)+\ 
(df_test.loc[df_test['model'] == 0, 'xgb_duo_model_predictions'] * 0.165)+\ 
(df_test.loc[df_test['model'] == 0, 'cb_duo_model_predictions'] * 0.165)
 df_test.loc[df_test['model'] == 1, 'blend_predictions'] =(df_test.loc[df_test['model'] == 1, 'lgb_solo_model_predictions'] * 0.17)+\ 
(df_test.loc[df_test['model'] == 1, 'xgb_solo_model_predictions'] * 0.165)+\ 
(df_test.loc[df_test['model'] == 1, 'cb_solo_model_predictions'] * 0.165)+\ 
(df_test.loc[df_test['model'] == 1, 'lgb_duo_model_predictions'] * 0.17)+\ 
(df_test.loc[df_test['model'] == 1, 'xgb_duo_model_predictions'] * 0.165)+\ 
(df_test.loc[df_test['model'] == 1, 'cb_duo_model_predictions'] * 0.165)
 df_test.loc[(df_test['model'] == 1.5)|(df_test['model'] == 2), 'blend_predictions'] =(df_test.loc[(df_test['model'] == 1.5)|(df_test['model'] == 2), 'lgb_solo_model_predictions'] * 0.14)+\ 
(df_test.loc[(df_test['model'] == 1.5)|(df_test['model'] == 2), 'xgb_solo_model_predictions'] * 0.13)+\ 
(df_test.loc[(df_test['model'] == 1.5)|(df_test['model'] == 2), 'cb_solo_model_predictions'] * 0.13)+\ 
(df_test.loc[(df_test['model'] == 1.5)|(df_test['model'] == 2), 'lgb_trio_model_predictions'] * 0.2)+\ 
(df_test.loc[(df_test['model'] == 1.5)|(df_test['model'] == 2), 'xgb_trio_model_predictions'] * 0.2)+\ 
(df_test.loc[(df_test['model'] == 1.5)|(df_test['model'] == 2), 'cb_trio_model_predictions'] * 0.2)
 df_test.loc[df_test['model'] == 3, 'blend_predictions'] =(df_test.loc[df_test['model'] == 3, 'lgb_solo_model_predictions'] * 0.14)+\ 
(df_test.loc[df_test['model'] == 3, 'xgb_solo_model_predictions'] * 0.13)+\ 
(df_test.loc[df_test['model'] == 3, 'cb_solo_model_predictions'] * 0.13)+\ 
(df_test.loc[df_test['model'] == 3, 'lgb_quad_model_predictions'] * 0.2)+\ 
(df_test.loc[df_test['model'] == 3, 'xgb_quad_model_predictions'] * 0.2)+\ 
(df_test.loc[df_test['model'] == 3, 'cb_quad_model_predictions'] * 0.2)
 df_test.loc[df_test['model'] == 4, 'blend_predictions'] =(df_test.loc[df_test['model'] == 4, 'lgb_solo_model_predictions'] * 0.034)+\ 
(df_test.loc[df_test['model'] == 4, 'xgb_solo_model_predictions'] * 0.033)+\ 
(df_test.loc[df_test['model'] == 4, 'cb_solo_model_predictions'] * 0.033)+\ 
(df_test.loc[df_test['model'] == 4, 'lgb_penta_model_predictions'] * 0.45)+\ 
(df_test.loc[df_test['model'] == 4, 'cb_penta_model_predictions'] * 0.45)
 df_test.loc[df_test['model'] == 0, 'blend_predictions'] = np.clip(df_test.loc[df_test['model'] == 0, 'blend_predictions'], 0, 1)
 df_test.loc[df_test['model'] == 1, 'blend_predictions'] = np.clip(df_test.loc[df_test['model'] == 1, 'blend_predictions'], 0, 1)
 df_test.loc[df_test['model'] == 1.5, 'blend_predictions'] = np.clip(df_test.loc[df_test['model'] == 1.5, 'blend_predictions'], 0, 3)
 df_test.loc[df_test['model'] == 2, 'blend_predictions'] = np.clip(df_test.loc[df_test['model'] == 2, 'blend_predictions'], 0, 3)
 df_test.loc[df_test['model'] == 3, 'blend_predictions'] = np.clip(df_test.loc[df_test['model'] == 3, 'blend_predictions'], 0, 5)
 df_test.loc[df_test['model'] == 4, 'blend_predictions'] = np.clip(df_test.loc[df_test['model'] == 4, 'blend_predictions'], 0, 10)
 df_test['blend_predictions'] = df_test['blend_predictions'].round().astype(np.uint8 )
def process_data(tweet, selected_text, sentiment, tokenizer):
tweet = " " + " ".join(str(tweet ).split())
 selected_text = " " + " ".join(str(selected_text ).split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids_orig = tok_tweet.ids
tweet_offsets = tok_tweet.offsets
target_idx = []
for j,(offset1, offset2)in enumerate(tweet_offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 targets_start = target_idx[0]
targets_end = target_idx[-1]
sentiment_id = {
'positive': 1313,
'negative': 2430,
'neutral': 7974
}
input_ids = [0] + [sentiment_id[sentiment]] + [2] + [2] + input_ids_orig + [2]
token_type_ids = [0, 0, 0, 0] + [0] *(len(input_ids_orig)+ 1)
 mask = [1] * len(token_type_ids)
 tweet_offsets = [(0, 0)] * 4 + tweet_offsets + [(0, 0)]
targets_start += 4
targets_end += 4
return {
'ids': input_ids,
'mask': mask,
'token_type_ids': token_type_ids,
'targets_start': targets_start,
'targets_end': targets_end,
'orig_tweet': tweet,
'orig_selected': selected_text,
'sentiment': sentiment,
'offsets': tweet_offsets
}<string_transform>
df_train.rename(columns={
'lgb_solo_model_oof': 'lgb_solo_model',
'xgb_solo_model_oof': 'xgb_solo_model',
'cb_solo_model_oof': 'cb_solo_model'
}, inplace=True)
 df_train['lgb_multi_model'] = 0
df_train.loc[df_train['model'] == 0, 'lgb_multi_model'] = df_train.loc[df_train['model'] == 0, 'lgb_duo_model_oof']
df_train.loc[df_train['model'] == 1, 'lgb_multi_model'] = df_train.loc[df_train['model'] == 1, 'lgb_duo_model_oof']
df_train.loc[df_train['model'] == 2, 'lgb_multi_model'] = df_train.loc[df_train['model'] == 2, 'lgb_trio_model_oof']
df_train.loc[df_train['model'] == 3, 'lgb_multi_model'] = df_train.loc[df_train['model'] == 3, 'lgb_quad_model_oof']
df_train.loc[df_train['model'] == 4, 'lgb_multi_model'] = df_train.loc[df_train['model'] == 4, 'lgb_penta_model_oof']
df_train['lgb_multi_model'] = df_train['lgb_multi_model'].astype(np.float32)
 df_train['xgb_multi_model'] = 0
df_train.loc[df_train['model'] == 0, 'xgb_multi_model'] = df_train.loc[df_train['model'] == 0, 'xgb_duo_model_oof']
df_train.loc[df_train['model'] == 1, 'xgb_multi_model'] = df_train.loc[df_train['model'] == 1, 'xgb_duo_model_oof']
df_train.loc[df_train['model'] == 2, 'xgb_multi_model'] = df_train.loc[df_train['model'] == 2, 'xgb_trio_model_oof']
df_train.loc[df_train['model'] == 3, 'xgb_multi_model'] = df_train.loc[df_train['model'] == 3, 'xgb_quad_model_oof']
df_train.loc[df_train['model'] == 4, 'xgb_multi_model'] = np.nan
df_train['xgb_multi_model'] = df_train['xgb_multi_model'].astype(np.float32)
 df_train['cb_multi_model'] = 0
df_train.loc[df_train['model'] == 0, 'cb_multi_model'] = df_train.loc[df_train['model'] == 0, 'cb_duo_model_oof']
df_train.loc[df_train['model'] == 1, 'cb_multi_model'] = df_train.loc[df_train['model'] == 1, 'cb_duo_model_oof']
df_train.loc[df_train['model'] == 2, 'cb_multi_model'] = df_train.loc[df_train['model'] == 2, 'cb_trio_model_oof']
df_train.loc[df_train['model'] == 3, 'cb_multi_model'] = df_train.loc[df_train['model'] == 3, 'cb_quad_model_oof']
df_train.loc[df_train['model'] == 4, 'cb_multi_model'] = df_train.loc[df_train['model'] == 4, 'cb_penta_model_oof']
df_train['cb_multi_model'] = df_train['cb_multi_model'].astype(np.float32)
 df_test.rename(columns={
'lgb_solo_model_predictions': 'lgb_solo_model',
'xgb_solo_model_predictions': 'xgb_solo_model',
'cb_solo_model_predictions': 'cb_solo_model'
}, inplace=True)
 df_test['lgb_multi_model'] = 0
df_test.loc[df_test['model'] == 0, 'lgb_multi_model'] = df_test.loc[df_test['model'] == 0, 'lgb_duo_model_predictions']
df_test.loc[df_test['model'] == 1, 'lgb_multi_model'] = df_test.loc[df_test['model'] == 1, 'lgb_duo_model_predictions']
df_test.loc[df_test['model'] == 2, 'lgb_multi_model'] = df_test.loc[df_test['model'] == 2, 'lgb_trio_model_predictions']
df_test.loc[df_test['model'] == 3, 'lgb_multi_model'] = df_test.loc[df_test['model'] == 3, 'lgb_quad_model_predictions']
df_test.loc[df_test['model'] == 4, 'lgb_multi_model'] = df_test.loc[df_test['model'] == 4, 'lgb_penta_model_predictions']
df_test['lgb_multi_model'] = df_test['lgb_multi_model'].astype(np.float32)
 df_test['xgb_multi_model'] = 0
df_test.loc[df_test['model'] == 0, 'xgb_multi_model'] = df_test.loc[df_test['model'] == 0, 'xgb_duo_model_predictions']
df_test.loc[df_test['model'] == 1, 'xgb_multi_model'] = df_test.loc[df_test['model'] == 1, 'xgb_duo_model_predictions']
df_test.loc[df_test['model'] == 2, 'xgb_multi_model'] = df_test.loc[df_test['model'] == 2, 'xgb_trio_model_predictions']
df_test.loc[df_test['model'] == 3, 'xgb_multi_model'] = df_test.loc[df_test['model'] == 3, 'xgb_quad_model_predictions']
df_test.loc[df_test['model'] == 4, 'xgb_multi_model'] = np.nan
df_test['xgb_multi_model'] = df_test['xgb_multi_model'].astype(np.float32)
 df_test['cb_multi_model'] = 0
df_test.loc[df_test['model'] == 0, 'cb_multi_model'] = df_test.loc[df_test['model'] == 0, 'cb_duo_model_predictions']
df_test.loc[df_test['model'] == 1, 'cb_multi_model'] = df_test.loc[df_test['model'] == 1, 'cb_duo_model_predictions']
df_test.loc[df_test['model'] == 2, 'cb_multi_model'] = df_test.loc[df_test['model'] == 2, 'cb_trio_model_predictions']
df_test.loc[df_test['model'] == 3, 'cb_multi_model'] = df_test.loc[df_test['model'] == 3, 'cb_quad_model_predictions']
df_test.loc[df_test['model'] == 4, 'cb_multi_model'] = df_test.loc[df_test['model'] == 4, 'cb_penta_model_predictions']
df_test['cb_multi_model'] = df_test['cb_multi_model'].astype(np.float32)
 PREDICTION_COLS = ['lgb_solo_model', 'xgb_solo_model', 'cb_solo_model', 'lgb_multi_model', 'xgb_multi_model', 'cb_multi_model']
df_train[PREDICTION_COLS].to_pickle('train_predictions.pkl')
 df_test[PREDICTION_COLS].to_pickle('test_predictions.pkl' )
class TweetDataset:
def __init__(self, tweet, sentiment, selected_text):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tokenizer = config.TOKENIZER
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
data = process_data(
 self.tweet[item],
self.selected_text[item],
self.sentiment[item],
self.tokenizer 
)
 return data<choose_model_class>
submission = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv')
 submission['open_channels'] = df_test['blend_predictions'].copy()
submission.to_csv('gbdt_blend_submission.csv', index=False, float_format='%.4f')
 print(f'GBDT Blend Submission
{submission["open_channels"].describe() }' )