Kaggle competition: INGV - Volcanic Eruption Prediction
<SOS> metric: MAE Kaggle data source: ingv-volcanic-eruption-prediction<train_model>
__seed = 666
__create_dataframes = True
__dataframe_size = 5000
__path_to_my_data = ".. /input/ingvchallengefeatures/"
__n_folds = 5
DecisionTreeClassifierModel = DecisionTreeClassifier(criterion='entropy',max_depth=5,random_state=33)
 DecisionTreeClassifierModel.fit(X_train, y_train)
 print('DecisionTreeClassifierModel Train Score is : ' , DecisionTreeClassifierModel.score(X_train, y_train))
print('DecisionTreeClassifierModel Test Score is : ' , DecisionTreeClassifierModel.score(X_test, y_test))
print('----------------------------------------------------')
 train_scores.append(DecisionTreeClassifierModel.score(X_train, y_train))
test_scores.append(DecisionTreeClassifierModel.score(X_test, y_test))<predict_on_test>
np.random.seed(__seed)
 pd.set_option('max_columns', 100)
 pd.set_option('max_rows', 200)
 __n_cores = psutil.cpu_count()
y_pred = RandomForestClassifierModel.predict(test_data )<save_to_csv>
y_train = pd.read_csv(".. /input/predict-volcanic-eruptions-ingv-oe/train.csv"
, index_col="segment_id"
, dtype={"time_to_eruption":np.int32}
, squeeze=True)
 print("Length of time_to_eruption serie : {}
The five first lines :".format(len(y_train)))
 print(y_train.head().map('{:,.0f}'.format))
gs = pd.read_csv('.. /input/titanic/gender_submission.csv')
 submission = pd.DataFrame({'PassengerId': gs.PassengerId, 'Survived': y_pred})
 submission.to_csv('my_submission.csv', index=False )<set_options>
print("Time to Eruption in Train
-------------------------
Min : {:,}
Median : {:,}
Max : {:,}".format(
 y_train.min() , y_train.median() , y_train.max()))
_temp = y_train.apply(lambda x:datetime.timedelta(milliseconds = x))
print("
Time to Eruption in Train
-------------------------
Min : {}
Median : {}
Max : {}".format(
 _temp.min() , _temp.median() , _temp.max()))
del _temp
warnings.filterwarnings("ignore")
 %matplotlib inline 
<drop_column>
sample_submission = pd.read_csv(".. /input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv")
 y_pred = sample_submission.set_index("segment_id")["time_to_eruption"].astype(np.int32)
 y_pred.head()
def drop_col_not_req(df, cols):
df.drop(cols, axis = 1, inplace = True )<groupby>
def make_file_submission(pred, filename = "submission.csv", verbose = False):
sample_submission["time_to_eruption"] = pred
sample_submission.loc[sample_submission["time_to_eruption"]<0, "time_to_eruption"]=0
sample_submission.to_csv(filename, index=False)
 if verbose:
print(sample_submission.head() )
def pclass_fare_category(df, Pclass_1_mean_fare, Pclass_2_mean_fare, Pclass_3_mean_fare):
if(df['Pclass'] == 1):
if(df['Fare'] <= Pclass_1_mean_fare):
return 'Pclass_1_Low_Fare'
else:
return 'Pclass_1_High_Fare'
elif(df['Pclass'] == 2):
if(df['Fare'] <= Pclass_2_mean_fare):
return 'Pclass_2_Low_Fare'
else:
return 'Pclass_2_High_Fare'
elif(df['Pclass'] == 3):
if(df['Fare'] <= Pclass_3_mean_fare):
return 'Pclass_3_Low_Fare'
else:
return 'Pclass_3_High_Fare'<train_on_grid>
def df_parallelize_run(func, t_split):
num_cores = np.min([__n_cores, len(t_split)])
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, t_split), axis=0)
 pool.close()
pool.join()
return df
def fill_missing_age(missing_age_train, missing_age_test):
missing_age_X_train = missing_age_train.drop(['Age'], axis = 1)
 missing_age_y_train = missing_age_train['Age']
missing_age_X_test = missing_age_test.drop(['Age'], axis = 1)
 gbm_reg = ensemble.GradientBoostingRegressor(random_state = 42)
 gbm_reg_param_grid = {'n_estimators': [2000], 'max_depth': [3], 'learning_rate': [0.01], 'max_features': [3]}
gbm_reg_grid = model_selection.GridSearchCV(gbm_reg, gbm_reg_param_grid, cv = 5, n_jobs = 25, verbose = 1, scoring = 'neg_mean_squared_error')
 gbm_reg_grid.fit(missing_age_X_train, missing_age_y_train)
 print("Age feature Best GB Params: " + str(gbm_reg_grid.best_params_))
print("Age feature Best GB Score: " + str(gbm_reg_grid.best_score_))
print("GB Train Error for 'Age' Feature Regressor: " + str(gbm_reg_grid.score(missing_age_X_train, missing_age_y_train)))
 missing_age_test['Age_GB'] = gbm_reg_grid.predict(missing_age_X_test)
 print(missing_age_test['Age_GB'][:4])
 lrf_reg = LinearRegression()
lrf_reg_param_grid = {'fit_intercept': [True], 'normalize': [True]}
lrf_reg_grid = model_selection.GridSearchCV(lrf_reg, lrf_reg_param_grid, cv = 5, n_jobs = 25, verbose = 1, scoring = 'neg_mean_squared_error')
 lrf_reg_grid.fit(missing_age_X_train, missing_age_y_train)
 print("Age feature Best LR Params: " + str(lrf_reg_grid.best_params_))
print("Age feature Best LR Score: " + str(lrf_reg_grid.best_score_))
print("LR Train Error for 'Age' Feature Regressor: " + str(lrf_reg_grid.score(missing_age_X_train, missing_age_y_train)))
 missing_age_test['Age_LRF'] = lrf_reg_grid.predict(missing_age_X_test)
 print(missing_age_test['Age_LRF'][:4])
 missing_age_test['Age'] = missing_age_test[['Age_GB', 'Age_LRF']].mean(axis = 1)
 print(missing_age_test['Age'][:4])
 drop_col_not_req(missing_age_test, ['Age_GB', 'Age_LRF'])
 return missing_age_test<train_on_grid>
def add_stats_on_the_row(df_stats, df_result = None):
input_list = list(itertools.product(*[list(df_stats.columns), list(df_stats.index.values)]))
columns = [f'{e[0]}_{str(e[1])}' for e in input_list]
res = pd.DataFrame(df_stats.values.reshape(1, df_stats.shape[0] * df_stats.shape[1], order="F"), columns = columns)
 if df_result is None:
return res
else:
return pd.concat([df_result, res], axis = 1)
 print("Little example, how I put statistics on a unique row :")
 df_ex = pd.DataFrame({'a':[100, 1, 2], 'b':[10, 20, 3], 'c':[20, 60, 7]})
 df_stats = add_stats_on_the_row(df_ex[["a", "c"]].agg(["min", "mean", "max"]))
print("Let's create a litte data frame
{}
And use this function to compute statistics :
{}".
 format(df_ex, df_stats))
del df_ex, df_stats
def get_top_n_features(titanic_train_data_X, titanic_train_data_y, top_n_features):
rf_est = RandomForestClassifier(random_state = 42)
 rf_param_grid = {'n_estimators' : [500], 'min_samples_split':[2, 3], 'max_depth':[20]}
rf_grid = model_selection.GridSearchCV(rf_est, rf_param_grid, n_jobs = 25, cv = 10, verbose = 1)
 rf_grid.fit(titanic_train_data_X, titanic_train_data_y)
 print("Top N Features Best RF Params: " + str(rf_grid.best_params_))
print("Top N Features Best RF Score: " + str(rf_grid.best_score_))
print("Top N Features RF Train Error: " + str(rf_grid.score(titanic_train_data_X, titanic_train_data_y)))
 feature_imp_sorted_rf = pd.DataFrame({'feature': list(titanic_train_data_X), 'importance': rf_grid.best_estimator_.feature_importances_} ).sort_values('importance', ascending = False)
 features_top_n_rf = feature_imp_sorted_rf.head(top_n_features)['feature']
print("Sample 25 Features from RF Classifier:")
 print(str(features_top_n_rf[:25]))
ada_est = ensemble.AdaBoostClassifier(random_state = 42)
 ada_param_grid = {'n_estimators' : [500], 'learning_rate': [0.5, 0.6]}
ada_grid = model_selection.GridSearchCV(ada_est, ada_param_grid, n_jobs = 25, cv = 10, verbose = 1)
 ada_grid.fit(titanic_train_data_X, titanic_train_data_y)
 print("Top N Features Best Ada Params: " + str(ada_grid.best_params_))
print("Top N Features Best Ada Score: " + str(ada_grid.best_score_))
print("Top N Features Ada Train Error: " + str(ada_grid.score(titanic_train_data_X, titanic_train_data_y)))
 feature_imp_sorted_ada = pd.DataFrame({'feature': list(titanic_train_data_X), 'importance': ada_grid.best_estimator_.feature_importances_} ).sort_values('importance', ascending = False)
 features_top_n_ada = feature_imp_sorted_ada.head(top_n_features)['feature']
print("Sample 25 Features from Ada Classifier:")
 print(str(features_top_n_ada[:25]))
et_est = ensemble.ExtraTreesClassifier(random_state = 42)
 et_param_grid = {'n_estimators' : [500], 'min_samples_split':[3, 4], 'max_depth':[15]}
et_grid = model_selection.GridSearchCV(et_est, et_param_grid, n_jobs = 25, cv = 10, verbose = 1)
 et_grid.fit(titanic_train_data_X, titanic_train_data_y)
 print("Top N Features Best ET Params: " + str(et_grid.best_params_))
print("Top N Features Best ET Score: " + str(et_grid.best_score_))
print("Top N Features ET Train Error: " + str(et_grid.score(titanic_train_data_X, titanic_train_data_y)))
 feature_imp_sorted_et = pd.DataFrame({'feature': list(titanic_train_data_X), 'importance': et_grid.best_estimator_.feature_importances_} ).sort_values('importance', ascending = False)
 features_top_n_et = feature_imp_sorted_et.head(top_n_features)['feature']
print("Sample 25 Features from ET Classifier:")
 print(str(features_top_n_et[:25]))
features_top_n = pd.concat([features_top_n_rf, features_top_n_ada, features_top_n_et], ignore_index = True ).drop_duplicates()
return features_top_n<split>
def lfeat_with_suff(suffix=""):
return [f"sensor_{i+1}{suffix}" for i in range(10)]
lfeat = lfeat_with_suff()
lfeat_abs = lfeat_with_suff("_abs")
 lfeat_fft_real = lfeat_with_suff("_fft_real")
 lfeat_fft_imag = lfeat_with_suff("_fft_imag")
 input_list = [range(10), ["min", "max"]]
list_result = list(itertools.product(*input_list))
def Stacking_Ensemble(models, X_train, y_train, X_test, n_folds):
X_train = np.array(X_train)
 y_train = np.array(y_train)
 X_test = np.array(X_test)
 Stacking_train = np.zeros(( X_train.shape[0],(len(models)* 2)))
 Stacking_test = np.zeros(( X_test.shape[0],(len(models)* 2)))
 Strat_KFold = model_selection.StratifiedKFold(n_splits = n_folds, shuffle = True, random_state = 42)
 skf = list(Strat_KFold.split(X_train, y_train))
for model_counter, model in enumerate(models):
print("Model_Counter" + str(model_counter))
S_test_temp_p1 = np.zeros(( titanic_test_data_X.shape[0], n_folds))
S_test_temp_p2 = np.zeros(( titanic_test_data_X.shape[0], n_folds))
for fold_counter,(tr_index, te_index)in enumerate(skf):
print("Fold : " + str(fold_counter))
X_tr = X_train[tr_index]
y_tr = y_train[tr_index]
X_te = X_train[te_index]
y_te = y_train[te_index]
model.fit(X_tr, y_tr)
 print("----------------------")
 print("Model : " + str(model_counter)+ "Fold : " + str(fold_counter)+ "Score is - " + str(model.score(X_tr, y_tr)))
 print("----------------------")
 tr_pred = model.predict_proba(X_te)
 Stacking_train[te_index, model_counter * 2] = tr_pred[:,0]
Stacking_train[te_index,(model_counter * 2)+ 1] = tr_pred[:,1]
te_pred = model.predict_proba(X_test)
 S_test_temp_p1[:, fold_counter] = te_pred[:,0]
S_test_temp_p2[:, fold_counter] = te_pred[:,1]
Stacking_test[:, model_counter * 2] = np.mean(S_test_temp_p1, axis = 1)
 Stacking_test[:,(model_counter * 2)+ 1] = np.mean(S_test_temp_p2, axis = 1)
 return(Stacking_train, Stacking_test)
<load_from_csv>
fs = 100
N = 60001
n = 256
max_f = 20
delta_f = fs / n
delta_t = n / fs / 2
def STFT_Features(segment_df, segment_id):
segment = [segment_id]
for sensor in segment_df.columns:
x = segment_df[sensor][:N]
if x.isna().sum() > 1000:
segment +=([np.NaN] * 10)
 continue
f, t, Z = scipy.signal.stft(x.fillna(0), fs = fs, window = 'hann', nperseg = n)
 f = f[:round(max_f/delta_f)+1]
Z = np.abs(Z[:round(max_f/delta_f)+1] ).T
th = Z.mean() * 1
Z_pow = Z.copy()
Z_pow[Z < th] = 0
Z_num = Z_pow.copy()
Z_num[Z >= th] = 1
Z_pow_sum = Z_pow.sum(axis = 0)
 Z_num_sum = Z_num.sum(axis = 0)
 A_pow = Z_pow_sum[round(10/delta_f):].sum()
A_num = Z_num_sum[round(10/delta_f):].sum()
BH_pow = Z_pow_sum[round(5/delta_f):round(8/delta_f)].sum()
BH_num = Z_num_sum[round(5/delta_f):round(8/delta_f)].sum()
BL_pow = Z_pow_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
BL_num = Z_num_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
C_pow = Z_pow_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
C_num = Z_num_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
D_pow = Z_pow_sum[round(2/delta_f):round(4/delta_f)].sum()
D_num = Z_num_sum[round(2/delta_f):round(4/delta_f)].sum()
segment += [A_pow, A_num, BH_pow, BH_num, BL_pow, BL_num, C_pow, C_num, D_pow, D_num]
features = [f"sensor_{i+1}_{f}" for i in range(10)for f in ["A_pow", "A_num", "BH_pow", "BH_num", "BL_pow", "BL_num"
, "C_pow", "C_num", "D_pow", "D_num"]]
df = pd.DataFrame(np.array([segment]), columns = ["segment_id"] + features)
 df["segment_id"] = df["segment_id"].astype(np.int32)
 return df
train_data_orig = pd.read_csv('.. /input/train.csv')
 test_data_orig = pd.read_csv('.. /input/test.csv' )<categorify>
def features_engineering_one_file(onefile, source):
sequence = pd.read_csv(f'/kaggle/input/predict-volcanic-eruptions-ingv-oe/{source}/{onefile}.csv')
 res = STFT_Features(sequence[lfeat], onefile)
 res = pd.concat([res, pd.DataFrame(sequence[lfeat].isna().sum().values.reshape(1, 10, order="F")
 , columns=[f"sensor_{i+1}_nan" for i in range(10)])], axis=1)
 sequence.fillna(0, inplace = True)
 sequence[lfeat_abs] = sequence[lfeat].abs()
ft = np.fft.fft(sequence[lfeat], axis=0)
 sequence[lfeat_fft_real] = np.real(ft)
 sequence[lfeat_fft_imag] = np.imag(ft)
 lfun = ["mean", "std", "min", "max", "mad", "skew", "kurtosis"]
lquantiles = [.01,.05,.1,.25,.5,.75,.9,.95,.99]
res = add_stats_on_the_row(sequence[lfeat + lfeat_abs + lfeat_fft_real + lfeat_fft_imag].agg(lfun), res)
 res = add_stats_on_the_row(sequence[lfeat + lfeat_abs + lfeat_fft_real + lfeat_fft_imag].quantile(lquantiles), res)
 return res
print(combined_train_test.groupby(['Survived', 'Embarked'])['Survived'].count())
 print(combined_train_test['PassengerId'].groupby(by = combined_train_test['Embarked'] ).count().sort_values(ascending = False))
print(combined_train_test['Fare'].groupby(by = combined_train_test['Embarked'] ).mean().sort_values(ascending = False))
if(combined_train_test['Embarked'].isnull().sum() != 0):
combined_train_test['Embarked'].fillna(combined_train_test['Embarked'].mode().iloc[0], inplace=True)
 combined_train_test.info()
emb_dummies_df = pd.get_dummies(combined_train_test['Embarked'],
prefix = combined_train_test[['Embarked']].columns[0])
 combined_train_test = pd.concat([combined_train_test, emb_dummies_df], axis = 1 )<categorify>
%%time
def create_data_for_model(df, source="train"):
func_partial_fe = partial(features_engineering_one_file, source = source)
 df_set = list()
df_set.append(df_parallelize_run(func_partial_fe, df.index[:__dataframe_size]))
df_set = pd.concat(df_set)
 df_set.reset_index(inplace = True, drop=True)
 df_set.set_index("segment_id", inplace=True)
 return df_set
if __create_dataframes:
print("Train dataset creation...")
 train_set = create_data_for_model(df = y_train)
 if __dataframe_size > 100:
print("Test dataset creation...")
 test_set = create_data_for_model(df = y_pred, source = "test")
 else:
train_set = pickle.load(open(__path_to_my_data + "ingv_train_set.pkl", "rb"))
test_set = pickle.load(open(__path_to_my_data + "ingv_test_set.pkl", "rb"))
pickle.dump(train_set, open("ingv_train_set.pkl", "wb"))
if __dataframe_size > 100:
pickle.dump(test_set, open("ingv_test_set.pkl", "wb"))
print(combined_train_test['Sex'].groupby(by = combined_train_test['Sex'] ).count().sort_values(ascending = False))
print(combined_train_test.groupby(['Survived', 'Sex'])['Survived'].count())

sex_dummies_df = pd.get_dummies(combined_train_test['Sex'],
prefix = combined_train_test[['Sex']].columns[0])
 combined_train_test = pd.concat([combined_train_test, sex_dummies_df], axis = 1 )<define_variables>
a=train_set.sum(axis=0, skipna=False)
 print("Features with NAN values :
{}".format(list(a[a.isna() ].index)) )
title_Dict = {}
title_Dict.update(dict.fromkeys(["Capt", "Col", "Major", "Dr", "Rev"], "Officer"))
title_Dict.update(dict.fromkeys(["Jonkheer", "Don", "Sir", "the Countess", "Dona", "Lady"], "Royalty"))
title_Dict.update(dict.fromkeys(["Mme", "Ms", "Mrs"], "Mrs"))
title_Dict.update(dict.fromkeys(["Mlle", "Miss"], "Miss"))
title_Dict.update(dict.fromkeys(["Mr", "Ms"], "Mr"))
title_Dict.update(dict.fromkeys(["Master"], "Master"))<categorify>
def other_fe(df_e):
df = df_e.copy()
lfeat = ['abs_0.5', 'abs_0.75', 'abs_0.95', 'abs_0.99', 'abs_max']
lfeat_new1 = ["q75_o_med", "q95_o_q75", "q99_o_q95", "max_o_q99"]
lfeat_new2 = ["q95_o_med", "q99_o_med", "max_o_med"]
for i in range(10):
for feat1, num, denom in zip(lfeat_new1, lfeat[1:], lfeat[:-1]):
df[f'sensor_{i+1}_{feat1}'] = df[f'sensor_{i+1}_{num}'] / df[f'sensor_{i+1}_{denom}']
df.loc[ df[f'sensor_{i+1}_{denom}'] == 0, f'sensor_{i+1}_{feat1}'] = 0
for feat2, num in zip(lfeat_new2, lfeat[2:]):
df[f'sensor_{i+1}_{feat2}'] = df[f'sensor_{i+1}_{num}'] / df[f'sensor_{i+1}_abs_0.5']
df.loc[ df[f'sensor_{i+1}_abs_0.5'] == 0, f'sensor_{i+1}_{feat2}'] = 0
df[f'sensor_{i+1}_q3_q1_abs'] = df[f'sensor_{i+1}_abs_0.75'] - df[f'sensor_{i+1}_abs_0.25']
df[f'sensor_{i+1}_q3_q1'] = df[f'sensor_{i+1}_0.75'] - df[f'sensor_{i+1}_0.25']
df[f'sensor_{i+1}_d9_d1_abs'] = df[f'sensor_{i+1}_abs_0.9'] - df[f'sensor_{i+1}_abs_0.1']
df[f'sensor_{i+1}_d9_d1'] = df[f'sensor_{i+1}_0.9'] - df[f'sensor_{i+1}_0.1']
df[[f"sensor_{i+1}_empty" for i in range(10)]]=0
for i in range(10):
df.loc[df[f"sensor_{i+1}_abs_max"]==0, f"sensor_{i+1}_empty"]=1
df.fillna(0, inplace=True)
 return df
train_set = other_fe(train_set)
 if __dataframe_size > 100:
test_set = other_fe(test_set)
 train_set["time_to_eruption"] = y_train
train_set.head()
combined_train_test['Title'] = combined_train_test['Title'].map(title_Dict)
 print(combined_train_test['Title'].groupby(by = combined_train_test['Title'] ).count().sort_values(ascending = False))
title_dummies_df = pd.get_dummies(combined_train_test['Title'],
prefix = combined_train_test[['Title']].columns[0])
 combined_train_test = pd.concat([combined_train_test, title_dummies_df], axis = 1 )<groupby>
a = train_set.nunique()
to_del = list(a[a<2].index)
 print("Features deleted :
{}".format(to_del))
train_set.drop(to_del, axis=1, inplace=True)
 test_set.drop(to_del, axis=1, inplace=True )
combined_train_test['Name_Length'] = combined_train_test['Name'].str.len()
print(combined_train_test['Name_Length'].groupby(by = combined_train_test['Name_Length'] ).count().sort_values(ascending = False)[:5] )<categorify>
def fit_and_predict_with_lgbm(df_train, params
, seed=__seed, features=None, X_test=None, check_feat_importance = True, n_folds=__n_folds
, early_stopping_rounds = 100):
folds = KFold(n_splits = n_folds, shuffle = True, random_state = seed)
 list_mae, list_r2 = [], []
predictions = None
feature_importance_df = pd.DataFrame()
feat_imp={}
params["random_state"] = seed + 1
params["bagging_seed"] = seed - 1
if not features is None:
lfeat = features
else:
lfeat = [f for f in df_train.columns if f not in ["time_to_eruption"]]
print("Train DF shape : {} rows and {} features".format(df_train.shape[0], len(lfeat)))
 if not X_test is None:
X_test = X_test[lfeat]
print("Test DF shape : {} rows and {} features".format(X_test.shape[0], X_test.shape[1]))
predictions = np.zeros(len(X_test))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(df_train)) :
print("Fold n°{}".format(fold_+1))
X_tr, X_val = df_train[lfeat].iloc[trn_idx], df_train[lfeat].iloc[val_idx]
y_tr, y_val = df_train['time_to_eruption'].iloc[trn_idx], df_train['time_to_eruption'].iloc[val_idx]
model = lgbm.LGBMRegressor(**params, n_estimators = 3000, n_jobs = -1)
 model.fit(X_tr, y_tr,
eval_set=[(X_tr, y_tr),(X_val, y_val)], eval_metric='mae',
verbose=1000, early_stopping_rounds = early_stopping_rounds)
 y_val_pred = model.predict(X_val, num_iteration=model.best_iteration_)
 _mae = mean_absolute_error(y_val_pred, y_val); list_mae.append(_mae)
 _r2 = r2_score(y_val_pred, y_val); list_r2.append(_r2)
 print("Scores on valid set for fold {} : MAE {:,.0f} & R2 {:.3f}".format(fold_+1, _mae, _r2))
if not X_test is None:
predictions += model.predict(X_test[lfeat], num_iteration=model.best_iteration_)/ folds.n_splits
if check_feat_importance:
fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = X_tr.columns
fold_importance_df["importance"] = model.feature_importances_[:len(X_tr.columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 for feat in X_tr.columns:
if fold_ == 0:
feat_imp[feat] = []
temp_df = X_val.copy()
temp_df[feat] = np.random.permutation(temp_df[feat])
 y_temp = model.predict(temp_df, num_iteration=model.best_iteration_)
 feat_imp[feat].append(mean_absolute_error(y_temp, y_val)- _mae)
 _mae = np.array(list_mae ).mean()
_r2 = np.array(list_r2 ).mean()
print("Mean scores on valid sets : MAE {:,.0f} & R2 {:.3f}".format(_mae, _r2))
feat_imp_df = None
if check_feat_importance:
feat_imp_df = pd.DataFrame(feat_imp ).transpose()
lfeat = [i for i in range(__n_folds)]
lfeat_sign = [f"sign_{i}" for i in range(n_folds)]
feat_imp_df[lfeat_sign] = 0
for mae_loss, mae_loss_sign in zip(lfeat, lfeat_sign):
feat_imp_df.loc[feat_imp_df[mae_loss]>0, mae_loss_sign] = 1
feat_imp_df["sum_sign"] = feat_imp_df[lfeat_sign].sum(axis=1)
 return {"pred":predictions, "feat_imp":feat_imp_df, "list_mae":list_mae, "mean_mae_valid":_mae
, "list_r2":list_r2, "mean_r2_valid":_r2, "plot_feat_imp":feature_importance_df}
params = {
'num_leaves': 28,
'min_data_in_leaf': 10,
'learning_rate': 0.15,
'max_bins': 50,
"feature_fraction": 0.5,
"bagging_freq": 1,
"bagging_fraction": 0.8,
"lambda_l1": 0.1,
"boosting": "gbdt",
'objective':'regression',
"metric": 'mae',
"verbosity": -1,
"nthread": -1,
}
combined_train_test['Name_Length_Category'] = combined_train_test['Name_Length'].map(name_len_category)
 print(combined_train_test['Name_Length_Category'].groupby(by = combined_train_test['Name_Length_Category'] ).count().sort_values(ascending = False))
le_fare = LabelEncoder()
le_fare.fit(np.array(['Very_Short_Name', 'Short_Name', 'Medium_Name', 'Long_Name', 'Very_High_Fare']))
combined_train_test['Name_Length_Category'] = le_fare.transform(combined_train_test['Name_Length_Category'])
 print(combined_train_test[['Name_Length_Category', 'Survived']].corr())
 first_name_dummies_df = pd.get_dummies(combined_train_test['Name_Length_Category'],
prefix = combined_train_test[['Name_Length_Category']].columns[0])
 combined_train_test = pd.concat([combined_train_test, first_name_dummies_df], axis = 1 )<categorify>
%%time
result1 = fit_and_predict_with_lgbm(train_set, params)
 feat_imp_df = result1["feat_imp"]
feats = list(feat_imp_df.loc[feat_imp_df["sum_sign"] == __n_folds].index)
 print("
There are {} features with no loss of MAE on the {} folds.
 ".format(len(feats), __n_folds))
combined_train_test['First_Name'] = combined_train_test['Name'].str.extract('^ (.+?),' ).str.strip()
print(combined_train_test['First_Name'].groupby(by = combined_train_test['First_Name'] ).count().sort_values(ascending = False)[:5])
 first_name_dummies_df = pd.get_dummies(combined_train_test['First_Name'],
prefix = combined_train_test[['First_Name']].columns[0])
 combined_train_test = pd.concat([combined_train_test, first_name_dummies_df], axis = 1 )<categorify>
%%time
result2 = fit_and_predict_with_lgbm(train_set, params, seed=__seed+10, features=feats, X_test = test_set)
 print("
Results summary :")
 print("CV n°1 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result1["mean_mae_valid"], result1["mean_r2_valid"]))
print("CV n°2 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result2["mean_mae_valid"], result2["mean_r2_valid"]))
feat_imp_df = result2["feat_imp"]
feats = list(feat_imp_df.loc[feat_imp_df["sum_sign"] == __n_folds].index)
 print("
There are {} features with no loss of MAE on the {} folds.
 ".format(len(feats), __n_folds))
make_file_submission(result2["pred"], "submission2.csv" )
combined_train_test['Last_Name'] = combined_train_test['Name'].str.split("\." ).str[1].str.strip()
combined_train_test['Last_Name'] = combined_train_test['Last_Name'].str.strip("\([^)]*\)")
 combined_train_test['Last_Name'].fillna(combined_train_test['Name'].str.split("\." ).str[1].str.strip())
 print(combined_train_test['Last_Name'].groupby(by = combined_train_test['Last_Name'] ).count().sort_values(ascending = False)[:5])
 last_name_dummies_df = pd.get_dummies(combined_train_test['Last_Name'],
prefix = combined_train_test[['Last_Name']].columns[0])
 combined_train_test = pd.concat([combined_train_test, last_name_dummies_df], axis = 1 )<categorify>
%%time
result3 = fit_and_predict_with_lgbm(train_set, params, seed=__seed+69, features=feats, X_test = test_set)
 print("
Results summary :")
 print("CV n°1 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result1["mean_mae_valid"], result1["mean_r2_valid"]))
print("CV n°2 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result2["mean_mae_valid"], result2["mean_r2_valid"]))
print("CV n°3 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result3["mean_mae_valid"], result3["mean_r2_valid"]))
feat_imp_df = result3["feat_imp"]
feats = list(feat_imp_df.loc[feat_imp_df["sum_sign"] == __n_folds].index)
 print("
There are {} features with no loss of MAE on the {} folds.
 ".format(len(feats), __n_folds))
make_file_submission(result3["pred"], "submission3.csv" )
combined_train_test['Original_Name'] = combined_train_test['Name'].str.split("\((.*?)\)" ).str[1].str.strip (<categorify>
%%time
result4 = fit_and_predict_with_lgbm(train_set, params, seed=__seed-41, features=feats, X_test = test_set)
 print("
Results summary :")
 print("CV n°1 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result1["mean_mae_valid"], result1["mean_r2_valid"]))
print("CV n°2 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result2["mean_mae_valid"], result2["mean_r2_valid"]))
print("CV n°3 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result3["mean_mae_valid"], result3["mean_r2_valid"]))
print("CV n°4 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result4["mean_mae_valid"], result4["mean_r2_valid"]))
feat_imp_df = result4["feat_imp"]
feats = list(feat_imp_df.loc[feat_imp_df["sum_sign"] == __n_folds].index)
 print("
There are {} features with no loss of MAE on the {} folds.
 ".format(len(feats), __n_folds))
make_file_submission(result4["pred"], "submission4.csv" )
if(combined_train_test['Fare'].isnull().sum() != 0):
combined_train_test['Fare'] = combined_train_test[['Fare']].fillna(combined_train_test.groupby('Pclass' ).transform('mean'))
combined_train_test.info()<categorify>
%%time
result5 = fit_and_predict_with_lgbm(train_set, params, seed=__seed-41, features=feats, X_test = test_set)
 print("
Results summary :")
 print("CV n°1 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result1["mean_mae_valid"], result1["mean_r2_valid"]))
print("CV n°2 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result2["mean_mae_valid"], result2["mean_r2_valid"]))
print("CV n°3 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result3["mean_mae_valid"], result3["mean_r2_valid"]))
print("CV n°4 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result4["mean_mae_valid"], result4["mean_r2_valid"]))
print("CV n°4 results - Mean MAE : {:,.0f}.Mean R2 : {:.3f}".format(result5["mean_mae_valid"], result5["mean_r2_valid"]))
feat_imp_df = result5["feat_imp"]
feats = list(feat_imp_df.loc[feat_imp_df["sum_sign"] == __n_folds].index)
 print("
There are {} features with no loss of MAE on the {} folds.
 ".format(len(feats), __n_folds))
make_file_submission(result5["pred"], "submission5.csv" )
combined_train_test['Group_Ticket'] = combined_train_test['Fare'].groupby(by = combined_train_test['Ticket'] ).transform('count')
 combined_train_test['Fare'] = combined_train_test['Fare']/combined_train_test['Group_Ticket']
combined_train_test.drop(['Group_Ticket'], axis = 1, inplace = True )<feature_engineering>
feat_imp_df[feat_imp_df["sum_sign"] != __n_folds]
<categorify><EOS> 
df = pd.concat([train_set.drop("time_to_eruption", axis=1), test_set], axis=0)
 ts = train_set.shape
li = df.iloc[:ts[0]].index.values
df["is_test"] = 1
df.loc[li, "is_test"] = 0
print(df["is_test"].value_counts() , "
")
 feat=[f"sensor_{i+1}_d9_d1" for i in range(10)]
tr, te, tr_y, te_y = train_test_split(df[feat], df["is_test"], test_size=0.33, random_state=__seed, shuffle=True)
 param = {'num_leaves': 30,
'min_data_in_leaf': 30,
'max_depth': -1,
'learning_rate': 0.1,
"min_child_samples": 20,
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9 ,
"bagging_seed": __seed,
"boosting": "gbdt",
'objective':'binary',
"metric": 'auc',
"verbosity": -1}
clf = lgbm.LGBMClassifier(**param, n_estimators = 1000, n_jobs = -1)
 clf.fit(tr, tr_y, eval_set = [(tr, tr_y),(te, te_y)], verbose=50, early_stopping_rounds = 100)
 feature_importances = pd.DataFrame(clf.feature_importances_
, index = tr.columns, columns=['importance'] ).sort_values('importance', ascending=False)
 feature_importances = feature_importances.reset_index()
feature_importances.columns = ['feature', 'importance']
fig, ax = plt.subplots(figsize =(18, 8))
sns.set()
plt.subplot(1, 1, 1);
sns.barplot(x="importance", y="feature", orient='h', data=feature_importances.head(50)) ;
plt.title('Feature Importance to detect Train or Test obs');