Kaggle competition: Mercedes-Benz Greener Manufacturing
<SOS> metric: R2Score Kaggle data source: mercedes-benz-greener-manufacturing<split>
pd.set_option('display.max_columns', None)
 warnings.filterwarnings(action="ignore")
 pd.options.display.max_seq_items = 8000
pd.options.display.max_rows = 8000
X_train, X_test, y_train, y_test = train_test_split(train_data, targets, test_size=0.2, random_state=42 )<define_search_space>
testing_set_scores = pd.DataFrame(data=[
['predection_output_stack_gen.csv(the final stacked model)', 0.55600],
['prediction_abr_lasso.csv', 0.54510],
['prediction_abr_tree.csv', 0.54816],
['prediction_gbr.csv', 0.54475],
['prediction_gbr_2.csv', 0.54645],
['prediction_lasso.csv', 0.54711],
['prediction_lgm.csv', 0.54998],
['prediction_lightgbm_2.csv', 0.54869],
['prediction_lightgbm_goss.csv', 0.55056],
['prediction_rf.csv', 0.55127],
['prediction_rf_2.csv', 0.55164],
['prediction_ridge.csv', 0.54308]
],columns=['model','score'] ).sort_index(axis=0, ascending=False)
 plt.barh(testing_set_scores['model'], testing_set_scores['score'])
 plt.xlim(0.5405,0.5576)
 for score, model in zip(testing_set_scores['score'], testing_set_scores['model']):
plt.text(x=score, y=model, s=score, ha='left', va='center', fontsize=9 )
param_grid = [
{
'clf__bootstrap':[True],
'clf__max_depth':[80],
'clf__max_features':[2],
'clf__min_samples_leaf':[4],
'clf__min_samples_split':[10],
'clf__n_estimators':[100]
},
]
grid_search = GridSearchCV(full_pipeline, param_grid, cv=5, scoring='roc_auc', refit=True)
 grid_search.fit(X_train, y_train)
 final_model = grid_search.best_estimator_
test_predictions = final_model.predict(X_test)
 accuracy = accuracy_score(y_test, test_predictions)
 precision = precision_score(y_test, test_predictions)
 recall = recall_score(y_test, test_predictions)
 f1 = f1_score(y_test, test_predictions)
 auc_roc = roc_auc_score(y_test, test_predictions)
 print("accuracy_score = {a}, precision_score = {b}, recall_score = {c}, f1_score = {d}, AUC ROC = {e}" 
.format(a=accuracy, b=precision, c=recall, d=f1, e=auc_roc))<compute_train_metric>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
train = pd.read_csv('/kaggle/input/train.csv', index_col=0)
 test = pd.read_csv('/kaggle/input/test.csv', index_col=0)
 print("train.shape:", train.shape, "; test.shape:",test.shape, "; columns only in train.csv:", set(train.columns)- set(test.columns))
train.head(n=2 )
y_scores = cross_val_predict(final_model, X_test, y_test, cv=5, method="predict_proba")
 fpr, tpr, thresholds = roc_curve(y_test, y_scores[:, 1] )<find_best_score>
def percent_missing(df):
data = pd.DataFrame(df)
 df_cols = list(pd.DataFrame(data))
dict_x = {}
for i in range(0, len(df_cols)) :
dict_x.update({df_cols[i]: round(data[df_cols[i]].isnull().mean() *100,2)})
 return dict_x
train_features_missing = percent_missing(train_features)
 print('Top 5 percent train_features missing', sorted(train_features_missing.items() , key=lambda x: x[1], reverse=True)[:5])
 test_features_missing = percent_missing(test_features)
 print('Top 5 percent train_features missing', sorted(test_features_missing.items() , key=lambda x: x[1], reverse=True)[:5] )
print('Best accuracy: %.3f' % grid_search.best_score_ )<save_to_csv>
category_feature_names = [fea for fea in train_features.columns if fea != 'y' and train_features[fea].dtype == 'object']
binary_feature_names = [fea for fea in train_features.columns if fea != 'y' and train_features[fea].dtype != 'object']
final_predictions = final_model.predict(test_data)
 output = pd.DataFrame({ 'PassengerId' : test_data["PassengerId"], 'Survived': final_predictions })
 output.to_csv('titanic-predictions.csv', index = False)
 output.head()<import_modules>
def encoding_features_with_expend(train_features, test_features, n_comp=12):
print('input: train shape={}; test shape={}'.format(train_features.shape, test_features.shape))
category_feature_names = [fea for fea in train_features.columns if train_features[fea].dtype == 'object']
oh_encoder = OneHotEncoder(handle_unknown='ignore')
 oh_encoder.fit(list(train_features[category_feature_names].values)+ list(test_features[category_feature_names].values))
train_cat_fea_oh_ndarray = oh_encoder.transform(train_features[category_feature_names] ).toarray()
test_cat_fea_oh_ndarray = oh_encoder.transform(test_features[category_feature_names] ).toarray()
train_cat_fea_oh = pd.DataFrame(data=train_cat_fea_oh_ndarray)
 test_cat_fea_oh = pd.DataFrame(data=test_cat_fea_oh_ndarray)
 print("train_cat_fea_oh.shape:", train_cat_fea_oh.shape)
 train_cat_fea_oh.index = train_features.index
test_cat_fea_oh.index = test_features.index
train_non_cat_fea = train_features.drop(category_feature_names, axis=1)
 test_non_cat_fea = test_features.drop(category_feature_names, axis=1)
 train_fea_oh = pd.concat([train_cat_fea_oh, train_non_cat_fea], axis=1)
 test_fea_oh = pd.concat([test_cat_fea_oh, test_non_cat_fea], axis=1)
 print('after one-hot: train shape={}; test shape={}'.format(train_fea_oh.shape, test_fea_oh.shape))
return train_fea_oh, test_fea_oh
train_features_oh, test_features_oh = encoding_features_with_expend(train_features, test_features )
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np<load_from_csv>
train_features, test_features = train_features_oh, test_features_oh
train = pd.read_csv('.. /input/titanic/train.csv' )<feature_engineering>
BASE_MODEL_KF = 7
STACK_MODEL_KF = 7
is_verify_code = False
X = train_features
y = train_labels
X_test = test_features
if is_verify_code:
X = X.head(n=BASE_MODEL_KF * STACK_MODEL_KF)
 y = y.head(n=BASE_MODEL_KF * STACK_MODEL_KF)
 X_test = X_test.head(n=BASE_MODEL_KF * STACK_MODEL_KF)
 X.shape, y.shape, X_test.shape
train.loc[train['Age']>=18,'Adult']=1
train.loc[train['Age']<18,'Adult']=0<feature_engineering>
kf = KFold(n_splits=BASE_MODEL_KF, random_state=37, shuffle=True)
 def cv_rmse(model, X=X, y=y):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kf))
return(rmse)
 def cv_r2(model, X=X, y=y, print_params=True):
r2 = cross_val_score(model, X, y, scoring="r2", cv=kf)
 print("---"*20)
 if print_params:
print("cv_r2: mean=", r2.mean() , "; std=", r2.std())
 print("--- model params: -------
")
 print(model)
 else:
print("cv_r2: ", r2.mean() , ";", r2.std())
 print("---"*20)
 return(r2)
 def plot_grid_cv_results(grid_with_train_score_returned):
grid = grid_with_train_score_returned
cv_results = grid.cv_results_
rst = pd.DataFrame(cv_results)[['mean_test_score', 'mean_train_score', 'std_test_score', 'std_train_score']]
rst[['mean_test_score', 'mean_train_score']].plot.bar(figsize=(50,5), grid=True)
 rst[['std_test_score', 'std_train_score']].plot.bar(figsize=(50,5), grid=True)
 print('---'*20)
 print(rst)
 print('---'*20)
 for i in range(0, len(cv_results['params'])) :
print(i, "\t:", cv_results['params'][i])
 print('---'*20)
 print('best_index:', grid.best_index_)
 print('best_score:', grid.best_score_)
 print('best_param:', grid.best_params_)
 return rst
train.loc[train['Fare']>=31,'Rich']=1
train.loc[train['Fare']<31,'Rich']=0<drop_column>
%%time
ridge = Pipeline([('ridge', Ridge(alpha=45, normalize=False, random_state=42)) ])
 normalize_alpha_list = list(range(25,35,2)) + list(range(35,45,1)) + list(range(45,55,2))
def grid_search_ridge(model=ridge):
params = [
{'ridge__alpha':normalize_alpha_list, 'ridge__normalize':[False]}
]
grid = GridSearchCV(model, params, cv=kf, scoring="r2", return_train_score=True)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
ridge = grid_search_ridge(ridge)
 cv_r2(ridge, X=X, y=y, print_params=False )
train.drop(['PassengerId','Name','SibSp','Parch','Ticket','Cabin','Age','Fare'],axis=1,inplace=True )<categorify>
%%time
rf = RandomForestRegressor(
 max_depth = 4,
n_estimators = 1000,
min_impurity_decrease = 0.175,
criterion = "mse",
min_samples_split = 2,
min_samples_leaf = 1,
min_weight_fraction_leaf = 0.02,
max_features = 0.88,
oob_score = True,
n_jobs = -1,
warm_start = False,
random_state = 42)
 def grid_search_random_forest(model = rf):
params = {"max_depth": [4],
"max_features": [0.88],
"min_weight_fraction_leaf":[0.02],
"min_impurity_decrease":[0.175, 0.2, 0.25],
"n_estimators":[1000]
}
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
rf = grid_search_random_forest()
train = pd.get_dummies(train,drop_first=True )<count_values>
%%time
rf_2 = RandomForestRegressor(
 n_estimators = 900,
max_features = None,
max_depth = 5,
min_samples_split = 5,
min_samples_leaf = 5,
oob_score = True,
random_state = 42)
 def grid_search_random_forest_2(model = rf_2):
params = {"n_estimators":[900],
"max_depth":[4,5,6],
"min_samples_split":[5],
"min_samples_leaf":[5]
}
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
rf_2 = grid_search_random_forest_2()
train.count()<data_type_conversions>
%%time
gbr = GradientBoostingRegressor(
 n_estimators = 1000,
learning_rate = 0.006,
max_features = 1.0,
min_impurity_decrease = 0.5,
subsample = 1,
max_depth = 3,
max_leaf_nodes = None,
min_samples_leaf = 1,
min_samples_split = 2,
loss = 'huber',
random_state = 42)
 def grid_search_gradient_boosting_regressor(model = gbr):
params = [{
"max_depth":[None], "max_leaf_nodes":[6]
}, {
"max_depth":[3], "max_leaf_nodes":[None]
}]
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
gbr = grid_search_gradient_boosting_regressor()
train = train.astype(np.int )<load_from_csv>
%%time
gbr_2 = GradientBoostingRegressor(
 n_estimators = 500,
learning_rate = 0.01,
max_depth = 4,
max_features = 0.6,
min_samples_leaf = 4,
min_samples_split = 40,
loss = 'huber',
random_state = 42)
 def grid_search_gradient_boosting_regressor_2(model = gbr_2):
params = {
"n_estimators":[500],
"learning_rate":[0.01],
"max_features":[0.8, 0.85],
"min_samples_split":[110, 120, 130],
"min_samples_leaf":[4],
}
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
gbr_2 = grid_search_gradient_boosting_regressor_2()
test = pd.read_csv('.. /input/titanic/test.csv' )<feature_engineering>
%%time
lightgbm = LGBMRegressor(
 boosting_type = 'gbdt',
objective = 'regression',
num_leaves = 31,
max_depth = 3,
learning_rate = 0.002,
n_estimators = 1000,
min_split_gain = 0.1,
max_bin = 40,
reg_alpha = 0,
reg_lambda = 0,
bagging_freq = 4,
bagging_fraction = 1.0,
feature_fraction = 1.0,
min_sum_hessian_in_leaf = 0.001,
n_jobs=-1, verbose = -1, random_state = 42)
 def grid_search_light_gbm(model = lightgbm):
params = {
"n_estimators": [1000],
"learning_rate": [0.002, 0.00225, 0.0025, 0.00275, 0.003],
"max_bin": [40],
"max_depth" : [3]
}
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
lightgbm = grid_search_light_gbm()
test.loc[test['Age']>=18,'Adult']=1
test.loc[test['Age']<18,'Adult']=0
test.loc[test['Fare']>=31,'Rich']=1
test.loc[test['Fare']<31,'Rich']=0<drop_column>
%%time
lightgbm_2 = LGBMRegressor(
 boosting_type = 'gbdt',
objective = 'regression',
num_leaves = 60,
max_depth = -1,
learning_rate = 0.01,
n_estimators = 350,
max_bin = 90,
bagging_fraction = 0.35,
reg_alpha = 0,
reg_lambda = 0,
bagging_freq = 4,
bagging_seed = 8,
feature_fraction = 0.8,
feature_fraction_seed = 8,
verbose = -1, random_state = 42)
 def grid_search_light_gbm_2(model = lightgbm_2):
params = [{
"num_leaves": [None], "max_depth":[3],
"feature_fraction": [0.8, 0.84, 0.88, 0.92]
}, {
"num_leaves": [6], "max_depth":[None],
"feature_fraction": [0.8, 0.84, 0.88, 0.92]
}]
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
lightgbm_2 = grid_search_light_gbm_2()
test.drop(['PassengerId','Name','SibSp','Parch','Ticket','Cabin','Age','Fare'],axis=1,inplace=True )<categorify>
%%time
abr_tree = AdaBoostRegressor(
 base_estimator = DecisionTreeRegressor(
 max_depth = 4,
min_samples_split = 2,
min_samples_leaf = 1,
min_weight_fraction_leaf = 0.0,
min_impurity_decrease = 0.4,
random_state = 53),
n_estimators = 1000,
learning_rate = 0.001,
loss = "exponential",
random_state = 42)
 def grid_search_ada_boost_tree_regressor(model = abr_tree):
params = {
"base_estimator__max_depth": [4],
"base_estimator__min_impurity_decrease":[0.55,0.6],
"base_estimator__min_weight_fraction_leaf":[0.0],
"learning_rate" : [0.001],
"loss" : ['exponential']
}
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
abr_tree = grid_search_ada_boost_tree_regressor()
test = pd.get_dummies(test,drop_first=True )<data_type_conversions>
%%time
lightgbm_goss = LGBMRegressor(
 boosting_type = 'goss',
objective = 'regression',
num_leaves = 31,
max_depth = 3,
learning_rate = 0.005,
n_estimators = 1000,
min_split_gain = 0.1,
max_bin = 40,
reg_alpha = 0,
reg_lambda = 0,
bagging_freq = 4,
bagging_fraction = 1.0,
feature_fraction = 1.0,
min_sum_hessian_in_leaf = 0.001,
n_jobs=-1, verbose = -1, random_state = 42 
)
 def grid_search_light_gbm_goss(model = lightgbm_goss):
params = {
"boosting_type":["goss"],
"n_estimators": [1000],
"learning_rate": [0.002],
"max_bin": [80],
"max_depth" : [3],
"num_leaves" : [6,7]
}
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
lightgbm_goss = grid_search_light_gbm_goss()
test = test.astype(np.int )<count_values>
%%time
alpha_list = [0.02, 0.025, 0.03, 0.05, 0.1, 0.5, 1, 1.5, 3, 6, 10, 13, 16, 20, 23, 26, 30, 33, 36, 40, 43, 46, 50]
abr_lasso = AdaBoostRegressor(
 base_estimator = LassoCV(alphas=alpha_list, normalize=False, random_state=42),
n_estimators = 24,
learning_rate = 0.0005,
loss = "exponential",
random_state = 42)
 def grid_search_abr_lasso(model = abr_lasso):
params = {
"n_estimators" : [18,12],
"learning_rate" : [0.0005, 0.001],
}
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
abr_lasso = grid_search_abr_lasso(abr_lasso)
test.count()<prepare_x_and_y>
%%time
alphas_list_1 = [0.001, 0.003, 0.005, 0.007, 0.009]
alphas_list_2 = [0.1, 0.3, 0.5, 0.7, 0.9]
alphas_list_3 = [1, 3, 5, 7, 9]
alphas_list_4 = [10, 13, 15, 17, 19, 21, 23, 25, 27, 29]
alphas_list_5 = [31, 33, 35, 37, 39, 41, 43, 45, 47, 49]
alphas_list_6 = [51, 53, 55, 57, 59, 61, 63, 65, 67, 70]
alphas_list_7 = [75, 80, 85, 90, 95, 100, 200, 400, 800, 1600]
kf_stack_gen = KFold(n_splits=STACK_MODEL_KF, random_state=37, shuffle=True)
 stack_gen = StackingCVRegressor(
 regressors =(gbr, rf, abr_tree, lightgbm, lasso, lightgbm_goss, rf_2, gbr_2, ridge, lightgbm_2, abr_lasso),
meta_regressor = make_pipeline(RobustScaler() , RidgeCV(scoring='r2', alphas=alphas_list_4)) ,
cv = kf_stack_gen,
n_jobs = 8,
use_features_in_secondary=False)
 def grid_search_stack_gen(meta_model_list, use_2nd_fea_list=[False], model=stack_gen):
params = [{'meta_regressor': meta_model_list, 'use_features_in_secondary': use_2nd_fea_list}]
grid_search = GridSearchCV(model, params, cv=3, scoring='r2', return_train_score=True)
 grid_search.fit(X, y)
 print("grid_search.best_params_: ", grid_search.best_params_)
 print("grid_search.best_estimator_: ", grid_search.best_estimator_)
 print("means test scores:")
 cvres = grid_search.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(mean_score, params)
 print("means training scores:")
 for mean_score, params in zip(cvres["mean_train_score"], cvres["params"]):
print(mean_score, params)
 print("details: -------------------------------------")
 print(cvres)
 return grid_search.best_estimator_
x_train = train.drop('Survived',axis=1)
 y_train = train['Survived']
x_test = test<choose_model_class>
%%time
print('fit stack_gen with full training data')
 stack_gen_model = stack_gen.fit(X,y)
 submission = pd.read_csv("/kaggle/input/sample_submission.csv")
 print("submission.shape:", submission.shape, "; submission.columns: ", submission.columns)
 print("X_test.shape:", X_test.shape)
 submission.iloc[:,1] = stack_gen_model.predict(X_test)
 submission.to_csv("predection_output_stack_gen.csv", index=False )
model = LogisticRegression()<train_model>
%%time
base_models_fitted = {}
def pred_testset_with_base_model() :
base_model_list = [('lasso', lasso),('ridge', ridge),('rf', rf),('gbr',gbr), 
('lgm', lightgbm),('abr_tree', abr_tree),('abr_lasso', abr_lasso), 
('lightgbm_goss', lightgbm_goss),('rf_2', rf_2),('gbr_2', gbr_2), 
('lightgbm_2', lightgbm_2)]
for name, model in base_model_list:
file_name = "prediction_" + name + ".csv"
trained_model = model.fit(X,y)
 submission.iloc[:,1] = trained_model.predict(X_test)
 submission.to_csv(file_name, index=False)
 base_models_fitted[name] = trained_model
print(file_name)
 pred_testset_with_base_model()