Kaggle competition: TMDB Box Office Prediction
df['r4t3'].value_counts()<count_values>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 sub_df = pd.read_csv('.. /input/sample_submission.csv' )
df['tamhog'].value_counts()<count_values>
train_df.isna().sum()
df['tamviv'].value_counts()<drop_column>
test_df.isna().sum()
total_features.remove('r4t3' )<count_values>
def genres_preprocessing(elem):
string = str(elem)
 str1 = string.replace(']','' ).replace('[','' ).replace('{','' ).replace('}','' ).replace (
df['dependency'].value_counts()<count_values>
genres_df.index.values
for g in genres_df.index.values:
train_df['isGenre_' + g] = train_df['genres_processed'].apply(lambda x: 1 if g in x else 0)
 test_df['isGenre_' + g] = test_df['genres_processed'].apply(lambda x: 1 if g in x else 0 )
df['SQBdependency'].value_counts()<count_values>
train_df["is_english_language"] = train_df.original_language.apply(lambda x: 1 if x == "en" else 0)
 test_df["is_english_language"] = test_df.original_language.apply(lambda x: 1 if x == "en" else 0)
 train_df.is_english_language = train_df.is_english_language.fillna(1)
 test_df.is_english_language = test_df.is_english_language.fillna(1)
 sns.set(rc={'figure.figsize':(12,8)})
 ax = sns.countplot(x="is_english_language", data=train_df )
df['edjefe'].value_counts()<count_values>
def production_companies_preprocessing(elem):
string = str(elem)
 str1 = string.replace(']','' ).replace('[','' ).replace('{','' ).replace('}','' ).replace(' ','' ).replace("name", "" ).replace("id", "" ).replace(":", "" ).replace("'", "")
 ll = str1.split(",")[0::2]
return ll
train_df["production_companies"] = train_df.production_companies.fillna('NoProductionCompany')
 train_df["production_companies"] = test_df.production_companies.fillna('NoProductionCompany')
 train_df["production_companies_processed"] = train_df.production_companies.apply(lambda elem: production_companies_preprocessing(elem))
test_df["production_companies_processed"] = test_df.production_companies.apply(lambda elem: production_companies_preprocessing(elem))
production_companies_dict = dict()
for production_company in train_df["production_companies_processed"]:
for elem in production_company:
if elem not in production_companies_dict:
production_companies_dict[elem] = 1
else:
production_companies_dict[elem] += 1
sns.set(rc={'figure.figsize':(12,8)})
 production_companies_df = pd.DataFrame.from_dict(production_companies_dict, orient='index')
 production_companies_df.columns = ["number_of_movies"]
production_companies_df = production_companies_df.sort_values(by="number_of_movies", ascending=False)
 production_companies_df.head(10 ).plot.bar()
plt.title("Number of films per production company" )
df['edjefa'].value_counts()<feature_engineering>
for g in production_companies_df.index.values:
train_df['isProductionCompany_' + g] = train_df['production_companies_processed'].apply(lambda x: 1 if g in x else 0)
 test_df['isProductionCompany_' + g] = test_df['production_companies_processed'].apply(lambda x: 1 if g in x else 0 )
df.loc[df.edjefa == 'yes', 'edjefa'] = 1
df.loc[df.edjefa == 'no', 'edjefa'] = 0 
<feature_engineering>
def production_countries_preprocessing(elem):
string = str(elem)
 str1 = string.replace(']','' ).replace('[','' ).replace('{','' ).replace('}','' ).replace(' ','' ).replace("name", "" ).replace("iso_3166_1", "" ).replace(":", "" ).replace("'", "")
 ll = str1.split(",")[0::2]
return ll
train_df["production_countries_processed"] = train_df.production_countries.fillna("NaN" ).apply(lambda elem: production_countries_preprocessing(elem))
test_df["production_countries_processed"] = test_df.production_countries.fillna("NaN" ).apply(lambda elem: production_countries_preprocessing(elem))
production_countries_dict = dict()
for production_country in train_df["production_countries_processed"]:
for elem in production_country:
if elem not in production_countries_dict:
production_countries_dict[elem] = 1
else:
production_countries_dict[elem] += 1
production_countries_df = pd.DataFrame.from_dict(production_countries_dict, orient='index')
 production_countries_df.columns = ["number_of_movies"]
production_countries_df = production_countries_df.sort_values(by="number_of_movies", ascending=False)
 production_countries_df.head(10 ).plot.bar()
plt.title("Number of films per production country" )
df.loc[df.edjefe == 'yes', 'edjefe'] = 1
df.loc[df.edjefe == 'no', 'edjefe'] = 0 
<data_type_conversions>
for c in production_countries_df.index.values:
train_df['isProductionCountry_' + c] = train_df['production_countries_processed'].apply(lambda x: 1 if c in x else 0)
 test_df['isProductionCountry_' + c] = test_df['production_countries_processed'].apply(lambda x: 1 if c in x else 0 )
df[['edjefa','edjefe']] = df[['edjefa','edjefe']].apply(pd.to_numeric)
<concatenate>
train_df["release_date"] = train_df["release_date"].fillna(train_df["release_date"].mode() [0])
 test_df["release_date"] = test_df["release_date"].fillna(test_df["release_date"].mode() [0])
 train_df['temp'] = train_df.release_date.apply(lambda x: datetime.strptime(x, '%m/%d/%y'))
train_df["month"] = train_df.temp.apply(lambda x: x.month)
 train_df["season"] = train_df["month"]%4
train_df["year"] = train_df.temp.apply(lambda x: x.year)
 train_df["day_of_week"] = train_df.temp.apply(lambda x: x.weekday() +1)
 train_df["week_of_year"] = train_df.temp.apply(lambda x: x.isocalendar() [1])
 train_df = train_df.drop(['temp'], axis=1)
 test_df['temp'] = test_df.release_date.apply(lambda x: datetime.strptime(x, '%m/%d/%y'))
test_df["month"] = test_df.temp.apply(lambda x: x.month)
 test_df["season"] = test_df["month"]%4
test_df["year"] = test_df.temp.apply(lambda x: x.year)
 test_df["day_of_week"] = test_df.temp.apply(lambda x: x.weekday() +1)
 test_df["week_of_year"] = test_df.temp.apply(lambda x: x.isocalendar() [1])
 test_df = test_df.drop(['temp'], axis=1)
 train_df["day_of_week"] = train_df["day_of_week"].fillna(train_df["day_of_week"].mode() [0])
 test_df["day_of_week"] = test_df["day_of_week"].fillna(test_df["day_of_week"].mode() [0])
 train_df["year"] = train_df["year"].fillna(train_df["year"].mode() [0])
 test_df["year"] = test_df["year"].fillna(test_df["year"].mode() [0])
 train_df["month"] = train_df["month"].fillna(train_df["month"].mode() [0])
 test_df["month"] = test_df["month"].fillna(test_df["month"].mode() [0])
 train_df["week_of_year"] = train_df["week_of_year"].fillna(train_df["week_of_year"].mode() [0])
 test_df["week_of_year"] = test_df["week_of_year"].fillna(test_df["week_of_year"].mode() [0])
 train_df["season"] = train_df["season"].fillna(train_df["season"].mode() [0])
 test_df["season"] = test_df["season"].fillna(test_df["season"].mode() [0])
 train_df[["release_date", "month", "year", "day_of_week", "week_of_year", "season"]].head()
total_features.append('edjefa')
 total_features.append('edjefe' )<count_values>
actors_dict = {}
size_of_actors = len(train_df)- train_df.cast.isna().sum()
for element in train_df[["revenue", "cast"]].values:
if type(element[1])== type(str()):
result = re.findall('name': '\w+\s*\w*', element[1])
 result = [x.replace("name': '", "")for x in result]
for actor in result:
if actor not in actors_dict:
actors_dict[actor] = element[0]
else:
actors_dict[actor] += element[0]
for actor in actors_dict:
actors_dict[actor] = actors_dict[actor]/size_of_actors
actors_df = pd.DataFrame.from_dict(actors_dict, orient='index', columns=["mean_movies_revenue"])
 actors_df.sort_values(by="mean_movies_revenue", ascending=False ).head(20 ).plot.bar()
df['abastaguadentro'].value_counts()<count_values>
def find_top_actor_from_cast(top_actor, element):
result = []
if type(element)== type(str()):
result = re.findall('name': '\w+\s*\w*', element)
 result = [x.replace("name': '", "")for x in result]
if top_actor in result:
return 1
else:
return 0
for top_actor in actors_df.sort_values(by="mean_movies_revenue", ascending=False ).head(10 ).index.values:
train_df["has_top_actor_"+ top_actor] = train_df.cast.apply(lambda element: find_top_actor_from_cast(top_actor, element))
test_df["has_top_actor_"+ top_actor] = test_df.cast.apply(lambda element: find_top_actor_from_cast(top_actor, element))
df['abastaguafuera'].value_counts()<count_values>
keywords_dict = {}
size_of_keywords = len(train_df)- train_df.Keywords.isna().sum()
for element in train_df[["revenue", "Keywords"]].values:
if type(element[1])== type(str()):
result = re.findall('name': '\w+\s*\w*', element[1])
 result = [x.replace("name': '", "")for x in result]
for key in result:
if key not in keywords_dict:
keywords_dict[key] = element[0]
else:
keywords_dict[key] += element[0]
for key in keywords_dict:
keywords_dict[key] = keywords_dict[key]/size_of_keywords
keywords_df = pd.DataFrame.from_dict(keywords_dict, orient='index', columns=["mean_movies_revenue"])
 keywords_df.sort_values(by="mean_movies_revenue", ascending=False ).head(10 ).plot.bar()
df['abastaguano'].value_counts()<groupby>
def find_top_keywords_from_cast(top_keyword, element):
result = []
if type(element)== type(str()):
result = re.findall('name': '\w+\s*\w*', element)
 result = [x.replace("name': '", "")for x in result]
if top_keyword in result:
return 1
else:
return 0
for top_keyword in keywords_df.sort_values(by="mean_movies_revenue", ascending=False ).head(10 ).index.values:
train_df["has_top_keyword_"+ top_keyword] = train_df.Keywords.apply(lambda element: find_top_keywords_from_cast(top_keyword, element))
test_df["has_top_keyword_"+ top_keyword] = test_df.Keywords.apply(lambda element: find_top_keywords_from_cast(top_keyword, element))
df_water_target = df.groupby('Target')[cols_water].sum().reset_index()
df_water_target<drop_column>
directors_dict = {}
size_of_crew = len(train_df)- train_df.crew.isna().sum()
for element in train_df[["revenue", "crew"]].values:
if type(element[1])== type(str()):
result = re.findall('Director', 'name': '\w+\s*\w*', element[1])
 result = [x.replace("Director', 'name': '", "")for x in result]
for key in result:
if key not in directors_dict:
directors_dict[key] = element[0]
else:
directors_dict[key] += element[0]
for key in directors_dict:
directors_dict[key] = directors_dict[key]/size_of_crew
directors_df = pd.DataFrame.from_dict(directors_dict, orient='index', columns=["mean_movies_revenue"])
 directors_df.sort_values(by="mean_movies_revenue", ascending=False ).head(10 ).plot.bar()
total_features.remove('abastaguano')
 total_features.remove('abastaguafuera')
 len(total_features )<count_values>
def find_top_directors_from_crew(top_director, element):
result = []
if type(element)== type(str()):
result = re.findall('Director', 'name': '\w+\s*\w*', element)
 result = [x.replace("Director', 'name': '", "")for x in result]
if top_director in result:
return 1
else:
return 0
for top_director in directors_df.sort_values(by="mean_movies_revenue", ascending=False ).head(10 ).index.values:
train_df["has_top_director_"+ top_director] = train_df.crew.apply(lambda element: find_top_directors_from_crew(top_director, element))
test_df["has_top_director_"+ top_director] = test_df.crew.apply(lambda element: find_top_directors_from_crew(top_director, element))
df['pisocemento'].value_counts()<groupby>
producers_dict = {}
size_of_crew = len(train_df)- train_df.crew.isna().sum()
for element in train_df[["revenue", "crew"]].values:
if type(element[1])== type(str()):
result = re.findall('Producer', 'name': '\w+\s*\w*', element[1])
 result = [x.replace("Producer', 'name': '", "")for x in result]
for key in result:
if key not in producers_dict:
producers_dict[key] = element[0]
else:
producers_dict[key] += element[0]
for key in producers_dict:
producers_dict[key] = producers_dict[key]/size_of_crew
producers_df = pd.DataFrame.from_dict(producers_dict, orient='index', columns=["mean_movies_revenue"])
 producers_df.sort_values(by="mean_movies_revenue", ascending=False ).head(10 ).plot.bar()
df_floor_target = df.groupby('Target')[cols_floor].sum().reset_index()
df_floor_target<drop_column>
def find_top_producers_from_crew(top_producer, element):
result = []
if type(element)== type(str()):
result = re.findall('Director', 'name': '\w+\s*\w*', element)
 result = [x.replace("Director', 'name': '", "")for x in result]
if top_producer in result:
return 1
else:
return 0
for top_producer in producers_df.sort_values(by="mean_movies_revenue", ascending=False ).head(10 ).index.values:
train_df["has_top_producer_"+ top_producer] = train_df.crew.apply(lambda element: find_top_producers_from_crew(top_producer, element))
test_df["has_top_producer_"+ top_producer] = test_df.crew.apply(lambda element: find_top_producers_from_crew(top_producer, element))
total_features.remove('pisonatur')
 total_features.remove('pisonotiene')
 total_features.remove('pisoother')
 len(total_features )<groupby>
for i, g in enumerate(genres_df.index.values):
genres_df.loc[g, "median_salary"] = train_df[train_df['isGenre_' + g]==1].revenue.median()
genres_df.sort_values(by=["number_of_movies", "median_salary"], ascending=False ).head(10 )
df_wall_target = df.groupby('Target')[cols_outside_wall].sum().reset_index()
df_wall_target<groupby>
for i, p in enumerate(genres_df.sort_values(by=["number_of_movies", "median_salary"], ascending=False ).head(10 ).index.values):
train_df['isTopGenre_' + p] = train_df['genres_processed'].apply(lambda x: 1 if p in x else 0)
 train_df['isTopGenre_Other'] = train_df['genres_processed'].apply(lambda x: 1 if p not in x else 0)
 test_df['isTopGenre_' + p] = test_df['genres_processed'].apply(lambda x: 1 if p in x else 0)
 test_df['isTopGenre_Other'] = test_df['genres_processed'].apply(lambda x: 1 if p not in x else 0 )
df_roof_target = df.groupby('Target')[cols_roof].sum().reset_index()
df_roof_target<groupby>
for i, c in enumerate(production_countries_df.index.values):
production_countries_df.loc[c, "median_salary"] = train_df[train_df['isProductionCountry_' + c]==1].revenue.median()
production_countries_df.sort_values(by=["number_of_movies", "median_salary"], ascending=False ).head(10 )
df_sani_target = df.groupby('Target')[cols_sanitary].sum().reset_index()
df_sani_target<count_missing_values>
for i, p in enumerate(production_countries_df.sort_values(by=["number_of_movies", "median_salary"], ascending=False ).head(10 ).index.values):
train_df['isTopProductionCountry_' + p] = train_df['production_countries_processed'].apply(lambda x: 1 if p in x else 0)
 test_df['isTopProductionCountry_' + p] = test_df['production_countries_processed'].apply(lambda x: 1 if p in x else 0 )
df['v2a1'].isna().sum()<count_values>
for i, p in enumerate(production_companies_df.index.values):
production_companies_df.loc[p, "median_salary"] = train_df[train_df['isProductionCompany_' + p]==1].revenue.median()
production_companies_df.sort_values(by=["number_of_movies", "median_salary"], ascending=False ).head(5 )
df['tipovivi3'].value_counts()<filter>
for i, p in enumerate(production_companies_df.sort_values(by=["number_of_movies", "median_salary"], ascending=False ).head(10 ).index.values):
train_df['isTopProductionCompany_' + p] = train_df['production_companies_processed'].apply(lambda x: 1 if p in x else 0)
 test_df['isTopProductionCompany_' + p] = test_df['production_companies_processed'].apply(lambda x: 1 if p in x else 0)
df.loc[(df['v2a1'].isna())&(df.tipovivi3 == 1)]<filter>
columns_for_training = ["log_budget", "log_popularity", "log_runtime", "day_of_week", "year", "month", "week_of_year", "season",
"num_genres", "num_of_production_countries", "log_num_of_cast", "log_num_of_male_cast", "log_num_of_female_cast", "has_collection",
"has_homepage", "has_tag", "is_english_language",
"log_num_of_crew", "log_num_of_male_crew", "log_num_of_female_crew",
"log_title_len", "log_overview_len", "log_tagline_len",
"log_num_of_directors", "log_num_of_producers", "log_num_of_editors", "log_num_of_art_crew", "log_num_of_sound_crew",
"log_num_of_costume_crew", "log_num_of_camera_crew", "log_num_of_visual_effects_crew", "log_num_of_lighting_crew",
"log_num_of_other_crew"]
columns_for_training.extend(train_df.select(lambda col: col.startswith('isTopGenre_'), axis=1 ).columns.values)
 columns_for_training.extend(train_df.select(lambda col: col.startswith('isTopProductionCompany_'), axis=1 ).columns.values)
 columns_for_training.extend(train_df.select(lambda col: col.startswith('isTopProductionCountry_'), axis=1 ).columns.values)
 columns_for_training.extend(train_df.select(lambda col: col.startswith('has_top_actor_'), axis=1 ).columns.values)
 columns_for_training.extend(train_df.select(lambda col: col.startswith('has_top_keyword_'), axis=1 ).columns.values)
 columns_for_training.extend(train_df.select(lambda col: col.startswith('has_top_director_'), axis=1 ).columns.values)
 columns_for_training.extend(train_df.select(lambda col: col.startswith('has_top_producer_'), axis=1 ).columns.values)
 columns_for_training
df[['v2a1','idhogar','parentesco1']].loc[(df.parentesco1 == 1)&(-df.v2a1.isna())&(df.tipovivi3==1)&(df.tipovivi1==0)].mean()<filter>
train_df[columns_for_training].isna().sum().sum()
df[['v2a1','idhogar','parentesco1']].loc[(df.parentesco1 != 1)&(-df.v2a1.isna())&(df.tipovivi3==1)&(df.tipovivi1==0)].mean()<set_options>
y = train_df['log_revenue']
X = train_df[columns_for_training]
kfold_splits = 5
df['v2a1'].fillna(120000, inplace=True )<count_missing_values>
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, random_state=42, test_size=0.3)
 predictions_test_xgb = np.zeros(len(test_df))
num_fold = 0
num_of_splits = kfold_splits
oof_rmse = 0
folds = KFold(n_splits=num_of_splits, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Fold:", num_fold)
 print()
clf_stra_xgb = xgb.XGBRegressor(n_estimators=10000, seed=42, nthread=-1)
 clf_stra_xgb.fit(xtrain_stra, ytrain_stra, eval_set=[(xtrain_stra, ytrain_stra),(xvalid_stra, yvalid_stra)],
early_stopping_rounds=1000, eval_metric='rmse', verbose=100)
 predictions_valid = clf_stra_xgb.predict(xvalid)
 rmse_valid = np.sqrt(mean_squared_error(yvalid, predictions_valid))
print("Fold",num_fold,"xvalid rmse:",rmse_valid)
 num_fold = num_fold + 1
oof_rmse += rmse_valid
predictions_test_xgb += clf_stra_xgb.predict(test_df[xtrain.columns])/num_of_splits
predictions_test_xgb = np.expm1(predictions_test_xgb)
 print()
print(predictions_test_xgb)
 print("OOF Out-of-fold rmse:", oof_rmse/num_of_splits)
 f, ax = plt.subplots(2, figsize=(12,7))
sns.set(rc={'figure.figsize':(9,86)})
 sns.distplot(train_df.revenue, ax=ax[0])
 ax[0].set_title("Train Set Revenue Histogram")
 sns.distplot(predictions_test_xgb, ax=ax[1])
 ax[1].set_title("Test Set Revenue Prediction Histogram")
 f.tight_layout()
xgb.plot_importance(clf_stra_xgb )
df['v2a1'].isna().sum()<prepare_x_and_y>
def bayesian_tuning(xtrain, ytrain):
bayes_cv_tuner = BayesSearchCV(
 estimator = xgb.XGBRegressor(
 nthread = -1,
objective = 'reg:linear',
verbosity=1,
random_state=42 
),
search_spaces = {
'learning_rate':(0.01, 1.0, 'log-uniform'),
'min_child_weight':(0, 10),
'n_estimators':(50, 300),
'max_depth':(2, 12),
'gamma':(1e-3, 1, 'log-uniform'),
'subsample':(0.01, 1.0, 'uniform'),
'colsample_bytree':(0.01, 1.0, 'uniform'),
'colsample_bylevel':(0.01, 1.0, 'uniform'),
'reg_lambda':(1e-1, 10, 'log-uniform'),
'reg_alpha':(1e-2, 1.0, 'log-uniform')
 },
cv = KFold(
 n_splits=kfold_splits,
shuffle=True,
random_state=42 
),
scoring = 'neg_mean_squared_error',
n_jobs = 2,
n_iter = 12,
verbose=0,
refit = True,
random_state = 42 
)
 def status_print(optim_result):

 all_models = pd.DataFrame(bayes_cv_tuner.cv_results_)
 best_params = pd.Series(bayes_cv_tuner.best_params_)
 print('Model
Best score: {}
Best params: {}
'.format(
 len(all_models),
np.round(bayes_cv_tuner.best_score_, 4),
bayes_cv_tuner.best_params_ 
))
result = bayes_cv_tuner.fit(xtrain, ytrain, callback = status_print)
 return result
result = bayesian_tuning(xtrain, ytrain )
X, y = df[total_features], df['Target']<split>
result.best_params_
seed = 42
test_size = 0.3
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=seed )<choose_model_class>
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, random_state=42, test_size=0.3)
 predictions_test_xgb_tuned = np.zeros(len(test_df))
num_fold = 0
oof_rmse = 0
num_of_splits = kfold_splits
folds = KFold(n_splits=num_of_splits, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Fold:", num_fold)
 print()
clf_stra_tuned_xgb = xgb.XGBRegressor(colsample_bytree = result.best_params_["colsample_bytree"],
colsample_bylevel = result.best_params_["colsample_bylevel"],
gamma=result.best_params_["gamma"],
learning_rate=result.best_params_["learning_rate"],
max_depth=result.best_params_["max_depth"],
min_child_weight=result.best_params_["min_child_weight"],
n_estimators=10000,
reg_alpha=result.best_params_["reg_alpha"],
reg_lambda=result.best_params_["reg_lambda"],
subsample=result.best_params_["subsample"],
seed=42,
nthread = -1)
 clf_stra_tuned_xgb.fit(xtrain_stra, ytrain_stra, eval_set=[(xtrain_stra, ytrain_stra),(xvalid_stra, yvalid_stra)],
early_stopping_rounds=1000, eval_metric='rmse', verbose=100)
 predictions_valid = clf_stra_tuned_xgb.predict(xvalid)
 rmse_valid = np.sqrt(mean_squared_error(yvalid, predictions_valid))
print("Fold",num_fold,"xvalid rmse:",rmse_valid)
 num_fold = num_fold + 1
oof_rmse += rmse_valid
predictions_test_xgb_tuned += clf_stra_tuned_xgb.predict(test_df[xtrain.columns])/num_of_splits
print()
predictions_test_xgb_tuned = np.expm1(predictions_test_xgb_tuned)
 print(predictions_test_xgb_tuned)
 print("OOF Out-of-fold rmse:", oof_rmse/num_of_splits)
 f, ax = plt.subplots(2, figsize=(12,7))
sns.set(rc={'figure.figsize':(9,86)})
 sns.distplot(train_df.revenue, ax=ax[0])
 ax[0].set_title("Train Set Revenue Histogram")
 sns.distplot(predictions_test_xgb_tuned, ax=ax[1])
 ax[1].set_title("Test Set Revenue Prediction Histogram")
 f.tight_layout()
xgb.plot_importance(clf_stra_tuned_xgb )
model1 = xgb.XGBClassifier(n_jobs=n_jobs)
 model1<train_model>
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, random_state=42, test_size=0.3)
 predictions_extra_trees_test = np.zeros(len(test_df))
num_fold = 0
num_of_splits = kfold_splits
oof_rmse = 0
folds = KFold(n_splits=num_of_splits, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Fold:", num_fold)
 print()
clf_extra_trees = ExtraTreesRegressor(n_estimators=100, random_state=42)
 clf_extra_trees.fit(xtrain_stra, ytrain_stra)
 predictions_valid = clf_extra_trees.predict(xvalid)
 rmse_valid = np.sqrt(mean_squared_error(yvalid, predictions_valid))
print("Fold" ,num_fold, "xvalid rmse:", rmse_valid)
 num_fold = num_fold + 1
oof_rmse += rmse_valid
predictions_extra_trees_test += clf_extra_trees.predict(test_df[xtrain.columns])/num_of_splits
predictions_extra_trees_test = np.expm1(predictions_extra_trees_test)
 print()
print(predictions_extra_trees_test)
 print()
print("OOF Out-of-fold rmse:", oof_rmse/num_of_splits)
 f, ax = plt.subplots(2, figsize=(12,7))
sns.set(rc={'figure.figsize':(9,14)})
 sns.distplot(train_df.revenue, ax=ax[0])
 ax[0].set_title("Train Set Revenue Histogram")
 sns.distplot(predictions_extra_trees_test, ax=ax[1])
 ax[1].set_title("Test Set Revenue Prediction Histogram")
 f.tight_layout()
train_model1 = model1.fit(X_train, y_train )<choose_model_class>
def bayesian_tuning_extra_trees(xtrain, ytrain):
bayes_cv_tuner = BayesSearchCV(
 estimator = ExtraTreesRegressor(
 random_state=42 
),
search_spaces = {
'n_estimators':(10, 500),
'max_depth':(1, 12),
'min_samples_split':(2, 20),
'min_samples_leaf':(1, 20)
 },
cv = KFold(
 n_splits=kfold_splits,
shuffle=True,
random_state=42 
),
scoring = 'neg_mean_squared_error',
n_jobs = 2,
n_iter = 12,
verbose = 0,
refit = True,
random_state = 42 
)
 def status_print(optim_result):

 all_models = pd.DataFrame(bayes_cv_tuner.cv_results_)
 best_params = pd.Series(bayes_cv_tuner.best_params_)
 print('Model
Best score: {}
Best params: {}
'.format(
 len(all_models),
np.round(bayes_cv_tuner.best_score_, 4),
bayes_cv_tuner.best_params_ 
))
result_extra_trees = bayes_cv_tuner.fit(xtrain, ytrain, callback = status_print)
 return result_extra_trees
result_extra_trees = bayesian_tuning_extra_trees(xtrain, ytrain )
model2 = xgb.XGBClassifier(n_estimators=100, max_depth=8, learning_rate=0.1, subsample=0.5, n_jobs=n_jobs)
 model2<train_model>
result_extra_trees.best_params_
train_model2 = model2.fit(X_train, y_train )<predict_on_test>
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, random_state=42, test_size=0.3)
 predictions_extra_trees_tuned_test = np.zeros(len(test_df))
num_fold = 0
num_of_splits = kfold_splits
oof_rmse = 0
folds = KFold(n_splits=num_of_splits, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Fold:", num_fold)
 print()
clf_extra_trees_tuned = ExtraTreesRegressor(random_state=42,
max_depth = result_extra_trees.best_params_['max_depth'],
min_samples_leaf = result_extra_trees.best_params_['min_samples_leaf'],
min_samples_split = result_extra_trees.best_params_['min_samples_split'],
n_estimators = result_extra_trees.best_params_['n_estimators'])
 clf_extra_trees_tuned.fit(xtrain_stra, ytrain_stra)
 predictions_valid = clf_extra_trees_tuned.predict(xvalid)
 rmse_valid = np.sqrt(mean_squared_error(yvalid, predictions_valid))
print("Fold" ,num_fold, "xvalid rmse:", rmse_valid)
 num_fold = num_fold + 1
oof_rmse += rmse_valid
predictions_extra_trees_tuned_test += clf_extra_trees_tuned.predict(test_df[xtrain.columns])/num_of_splits
predictions_extra_trees_tuned_test = np.expm1(predictions_extra_trees_tuned_test)
 print()
print(predictions_extra_trees_tuned_test)
 print()
print("OOF Out-of-fold rmse:", oof_rmse/num_of_splits)
 f, ax = plt.subplots(2, figsize=(12,7))
sns.set(rc={'figure.figsize':(9,14)})
 sns.distplot(train_df.revenue, ax=ax[0])
 ax[0].set_title("Train Set Revenue Histogram")
 sns.distplot(predictions_extra_trees_tuned_test, ax=ax[1])
 ax[1].set_title("Test Set Revenue Prediction Histogram")
 f.tight_layout()
pred1 = train_model1.predict(X_test)
 pred2 = train_model2.predict(X_test)
 print('Model 1 XGboost Report %r' %(classification_report(y_test, pred1)))
 print('Model 2 XGboost Report %r' %(classification_report(y_test, pred2)) )<compute_test_metric>
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, random_state=42, test_size=0.3)
 predictions_random_forest_test = np.zeros(len(test_df))
num_fold = 0
num_of_splits = kfold_splits
oof_rmse = 0
folds = KFold(n_splits=num_of_splits, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Fold:", num_fold)
 print()
clf_random_forest = RandomForestRegressor(random_state=42, n_estimators = 100)
 clf_random_forest.fit(xtrain_stra, ytrain_stra)
 predictions_valid = clf_random_forest.predict(xvalid)
 rmse_valid = np.sqrt(mean_squared_error(yvalid, predictions_valid))
print("Fold" ,num_fold, "xvalid rmse:", rmse_valid)
 num_fold = num_fold + 1
oof_rmse += rmse_valid
predictions_random_forest_test += clf_random_forest.predict(test_df[xtrain.columns])/num_of_splits
predictions_random_forest_test = np.expm1(predictions_random_forest_test)
 print()
print(predictions_random_forest_test)
 print()
print("OOF Out-of-fold rmse:", oof_rmse/num_of_splits)
 f, ax = plt.subplots(2, figsize=(12,7))
sns.set(rc={'figure.figsize':(9,14)})
 sns.distplot(train_df.revenue, ax=ax[0])
 ax[0].set_title("Train Set Revenue Histogram")
 sns.distplot(predictions_random_forest_test, ax=ax[1])
 ax[1].set_title("Test Set Revenue Prediction Histogram")
 f.tight_layout()
print("Accuracy for model 1: %.2f" %(accuracy_score(y_test, pred1)* 100))
print("Accuracy for model 2: %.2f" %(accuracy_score(y_test, pred2)* 100))<choose_model_class>
def bayesian_tuning_random_forest(xtrain, ytrain):
bayes_cv_tuner = BayesSearchCV(
 estimator = RandomForestRegressor(
 random_state=42 
),
search_spaces = {
'n_estimators':(10, 500),
'max_depth':(1, 10),
'min_samples_split':(2, 20),
'min_samples_leaf':(1, 20)
 },
cv = KFold(
 n_splits=kfold_splits,
shuffle=True,
random_state=42 
),
scoring = 'neg_mean_squared_error',
n_jobs = 2,
n_iter = 12,
verbose = 0,
refit = True,
random_state = 42 
)
 def status_print(optim_result):

 all_models = pd.DataFrame(bayes_cv_tuner.cv_results_)
 best_params = pd.Series(bayes_cv_tuner.best_params_)
 print('Model
Best score: {}
Best params: {}
'.format(
 len(all_models),
np.round(bayes_cv_tuner.best_score_, 4),
bayes_cv_tuner.best_params_ 
))
result_random_forest = bayes_cv_tuner.fit(xtrain, ytrain, callback = status_print)
 return result_random_forest
result_random_forest = bayesian_tuning_random_forest(xtrain, ytrain )
model3 = xgb.XGBClassifier(
 learning_rate =0.1,
n_estimators=1000,
max_depth=5,
min_child_weight=1,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
n_jobs=n_jobs,
scale_pos_weight=1,
seed=27 )<predict_on_test>
result_random_forest.best_params_
train_model3 = model3.fit(X_train, y_train)
 pred3 = train_model3.predict(X_test)
 print("Accuracy for model 3: %.2f" %(accuracy_score(y_test, pred3)* 100))<compute_test_metric>
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, random_state=42, test_size=0.3)
 predictions_random_forest_tuned_test = np.zeros(len(test_df))
num_fold = 0
num_of_splits = kfold_splits
oof_rmse = 0
folds = KFold(n_splits=num_of_splits, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Fold:", num_fold)
 print()
clf_random_forest_tuned = RandomForestRegressor(random_state=42,
n_estimators = result_random_forest.best_params_['n_estimators'],
min_samples_leaf = result_random_forest.best_params_['min_samples_leaf'],
min_samples_split = result_random_forest.best_params_['min_samples_split'])
 clf_random_forest_tuned.fit(xtrain_stra, ytrain_stra)
 predictions_valid = clf_random_forest_tuned.predict(xvalid)
 rmse_valid = np.sqrt(mean_squared_error(yvalid, predictions_valid))
print("Fold xvalid rmse:", rmse_valid)
 oof_rmse += rmse_valid
predictions_random_forest_tuned_test += clf_random_forest_tuned.predict(test_df[xtrain.columns])/num_of_splits
predictions_random_forest_tuned_test = np.expm1(predictions_random_forest_tuned_test)
 print()
print(predictions_random_forest_test)
 print()
print("OOF Out-of-fold rmse:", oof_rmse/num_of_splits)
 f, ax = plt.subplots(2, figsize=(12,7))
sns.set(rc={'figure.figsize':(9,14)})
 sns.distplot(train_df.revenue, ax=ax[0])
 ax[0].set_title("Train Set Revenue Histogram")
 sns.distplot(predictions_random_forest_tuned_test, ax=ax[1])
 ax[1].set_title("Test Set Revenue Prediction Histogram")
 f.tight_layout()
print('Model 3 XGboost Report %r' %(classification_report(y_test, pred3)) )<set_options>
params = {
"metric": 'rmse',
"verbosity": -1
}
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, random_state=42, test_size=0.3)
 predictions_test_lgb = np.zeros(len(test_df))
num_fold = 0
oof_rmse = 0
num_of_splits = kfold_splits
folds = KFold(n_splits=num_of_splits, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Fold:", num_fold)
 print()
model_lgb = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model_lgb.fit(xtrain_stra, ytrain_stra,
eval_set=[(xtrain_stra, ytrain_stra),(xvalid_stra, yvalid_stra)], eval_metric='rmse',
verbose=100, early_stopping_rounds=1000)
 predictions_valid = model_lgb.predict(xvalid)
 rmse_valid = np.sqrt(mean_squared_error(yvalid, predictions_valid))
print("Fold" ,num_fold, "xvalid rmse:", rmse_valid)
 num_fold = num_fold + 1
oof_rmse += rmse_valid
predictions_test_lgb += model_lgb.predict(test_df[xtrain.columns])/num_of_splits
predictions_test_lgb = np.expm1(predictions_test_lgb)
 print()
print(predictions_test_lgb)
 print("OOF Out-of-fold rmse:", oof_rmse/num_of_splits)
 f, ax = plt.subplots(2, figsize=(12,7))
sns.set(rc={'figure.figsize':(9,14)})
 sns.distplot(train_df.revenue, ax=ax[0])
 ax[0].set_title("Train Set Revenue Histogram")
 sns.distplot(predictions_test_lgb, ax=ax[1])
 ax[1].set_title("Test Set Revenue Prediction Histogram")
 f.tight_layout()
gc.collect()<define_search_space>
def bayesian_tuning_lgb(xtrain, ytrain):
bayes_cv_tuner = BayesSearchCV(
 estimator = lgb.LGBMRegressor(
 boosting_type='gbdt', n_jobs=2, nthread = 4, verbose=-1 
),
search_spaces = {
'num_leaves':(10, 100),
'min_data_in_leaf':(10, 100),
'n_estimators':(50, 100),
'max_depth':(3, 12),
'learning_rate':(0.01, 0.2, 'log-uniform'),
"feature_fraction":(0.1, 1, 'uniform'),
"bagging_fraction":(0.1, 1, 'uniform'),
'lambda_l1':(0.1, 1, 'log-uniform'),
'lambda_l2':(0.1, 1, 'log-uniform')
 },
cv = KFold(
 n_splits=kfold_splits,
shuffle=True,
random_state=42 
),
scoring = 'neg_mean_squared_error',
n_jobs = 1,
n_iter = 12,
verbose = 0,
refit = True,
random_state = 42 
)
 def status_print(optim_result):

 all_models = pd.DataFrame(bayes_cv_tuner.cv_results_)
 best_params = pd.Series(bayes_cv_tuner.best_params_)
 print('Model
Best score: {}
Best params: {}
'.format(
 len(all_models),
np.round(bayes_cv_tuner.best_score_, 4),
bayes_cv_tuner.best_params_ 
))
result_lgbm = bayes_cv_tuner.fit(xtrain, ytrain, callback = status_print)
 return result_lgbm
result_lgbm = bayesian_tuning_lgb(xtrain, ytrain )
parameters = {
'n_estimators': [100],
'max_depth': [6, 9],
'subsample': [0.9, 1.0],
'colsample_bytree': [0.9, 1.0],
} 
<choose_model_class>
result_lgbm.best_params_
grid = GridSearchCV(model3,
parameters, n_jobs=n_jobs,
scoring="neg_log_loss",
cv=3)
 grid<compute_test_metric>
params = {
'num_leaves': result_lgbm.best_params_["num_leaves"],
'min_data_in_leaf': result_lgbm.best_params_["min_data_in_leaf"],
'max_depth': result_lgbm.best_params_["max_depth"],
'learning_rate': result_lgbm.best_params_["learning_rate"],
"boosting": "gbdt",
"feature_fraction": result_lgbm.best_params_["feature_fraction"],
"bagging_freq": 1,
"bagging_fraction": result_lgbm.best_params_["bagging_fraction"],
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": result_lgbm.best_params_["lambda_l1"],
"lambda_l2": result_lgbm.best_params_["lambda_l2"],
"verbosity": -1
}
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, random_state=42, test_size=0.3)
 predictions_test_lgb_tuned = np.zeros(len(test_df))
num_fold = 0
oof_rmse = 0
num_of_splits = kfold_splits
folds = KFold(n_splits=num_of_splits, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Fold:", num_fold)
 print()
model_lgb_tuned = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model_lgb_tuned.fit(xtrain_stra, ytrain_stra,
eval_set=[(xtrain_stra, ytrain_stra),(xvalid_stra, yvalid_stra)], eval_metric='rmse',
verbose=100, early_stopping_rounds=1000)
 predictions_valid = model_lgb_tuned.predict(xvalid)
 rmse_valid = np.sqrt(mean_squared_error(yvalid, predictions_valid))
print("Fold" ,num_fold, "xvalid rmse:", rmse_valid)
 num_fold = num_fold + 1
oof_rmse += rmse_valid
predictions_test_lgb_tuned += model_lgb_tuned.predict(test_df[xtrain.columns])/num_of_splits
predictions_test_lgb_tuned = np.expm1(predictions_test_lgb_tuned)
 print()
print(predictions_test_lgb_tuned)
 print("OOF Out-of-fold rmse:", oof_rmse/num_of_splits)
 f, ax = plt.subplots(2, figsize=(12,7))
sns.set(rc={'figure.figsize':(9,14)})
 sns.distplot(train_df.revenue, ax=ax[0])
 ax[0].set_title("Train Set Revenue Histogram")
 sns.distplot(predictions_test_lgb_tuned, ax=ax[1])
 ax[1].set_title("Test Set Revenue Prediction Histogram")
 f.tight_layout()
<set_options>
max_selected_features = 200
sel = SelectFromModel(clf_stra_xgb, max_features = max_selected_features, threshold=0.005, prefit=True)
 feature_idx = sel.get_support()
selected_features_xgb = X.columns[feature_idx]
gc.collect()<define_variables>
xtrain, xvalid, ytrain, yvalid = train_test_split(X[selected_features_xgb], y, random_state=42, test_size=0.3)
 start_time = time.time()
predictions_test_xgb_fs = np.zeros(len(test_df))
num_fold = 0
oof_rmse = 0
num_of_splits = kfold_splits
folds = KFold(n_splits=num_of_splits, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Fold:", num_fold)
 print()
clf_stra_fs_xgb = xgb.XGBRegressor(n_estimators=10000, seed=42, nthread = -1)
 clf_stra_fs_xgb.fit(xtrain_stra, ytrain_stra, eval_set=[(xtrain_stra, ytrain_stra),(xvalid_stra, yvalid_stra)],
early_stopping_rounds=1000, eval_metric='rmse', verbose=100)
 predictions_valid = clf_stra_fs_xgb.predict(xvalid)
 rmse_valid = np.sqrt(mean_squared_error(yvalid, predictions_valid))
print("Fold" ,num_fold, "xvalid rmse:", rmse_valid)
 num_fold = num_fold + 1
oof_rmse += rmse_valid
predictions_test_xgb_fs += clf_stra_fs_xgb.predict(test_df[xtrain.columns])/num_of_splits
predictions_test_xgb_fs = np.expm1(predictions_test_xgb_fs)
 print(predictions_test_xgb_fs)
 print("OOF Out-of-fold rmse:", oof_rmse/num_of_splits)
 f, ax = plt.subplots(2, figsize=(12,7))
sns.set(rc={'figure.figsize':(9,14)})
 sns.distplot(train_df.revenue, ax=ax[0])
 ax[0].set_title("Train Set Revenue Histogram")
 sns.distplot(predictions_test_xgb_fs, ax=ax[1])
 ax[1].set_title("Test Set Revenue Prediction Histogram")
 f.tight_layout()
xgb.plot_importance(clf_stra_fs_xgb )
less_imp_features = ['estadocivil1','instlevel9','techocane','parentesco10','v14a',
'parentesco11','parentesco5','paredother','parentesco7','noelec',
'elimbasu4','elimbasu6'] 
<drop_column>
result = bayesian_tuning(xtrain, ytrain )
for f in less_imp_features:
if f in total_features:
total_features.remove(f)
 len(total_features )<prepare_x_and_y>
xtrain, xvalid, ytrain, yvalid = train_test_split(X[selected_features_xgb], y, random_state=42, test_size=0.3)
 predictions_test_xgb_fs_tuned = np.zeros(len(test_df))
num_fold = 0
oof_rmse = 0
num_of_splits = kfold_splits
folds = KFold(n_splits=num_of_splits, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Fold:", num_fold)
 print()
clf_stra_fs_tuned_xgb = xgb.XGBRegressor(colsample_bytree = result.best_params_["colsample_bytree"],
gamma=result.best_params_["gamma"],
learning_rate=result.best_params_["learning_rate"],
max_depth=result.best_params_["max_depth"],
min_child_weight=result.best_params_["min_child_weight"],
n_estimators=10000,
reg_alpha=result.best_params_["reg_alpha"],
reg_lambda=result.best_params_["reg_lambda"],
subsample=result.best_params_["subsample"],
seed=42,
nthread = -1)
 clf_stra_fs_tuned_xgb.fit(xtrain_stra, ytrain_stra, eval_set=[(xtrain_stra, ytrain_stra),(xvalid_stra, yvalid_stra)],
early_stopping_rounds=1000, eval_metric='rmse', verbose=100)
 predictions_valid = clf_stra_fs_tuned_xgb.predict(xvalid)
 rmse_valid = np.sqrt(mean_squared_error(yvalid, predictions_valid))
print("Fold" ,num_fold, "xvalid rmse:", rmse_valid)
 num_fold = num_fold + 1
oof_rmse += rmse_valid
predictions_test_xgb_fs_tuned += clf_stra_fs_tuned_xgb.predict(test_df[xtrain.columns])/num_of_splits
predictions_test_xgb_fs_tuned = np.expm1(predictions_test_xgb_fs_tuned)
 print(predictions_test_xgb_fs_tuned)
 print("OOF Out-of-fold rmse:", oof_rmse/num_of_splits)
 f, ax = plt.subplots(2, figsize=(12,7))
sns.set(rc={'figure.figsize':(9,14)})
 sns.distplot(train_df.revenue, ax=ax[0])
 ax[0].set_title("Train Set Revenue Histogram")
 sns.distplot(predictions_test_xgb_fs_tuned, ax=ax[1])
 ax[1].set_title("Test Set Revenue Prediction Histogram")
 f.tight_layout()
xgb.plot_importance(clf_stra_fs_tuned_xgb )
X, y = df[total_features], df['Target']<split>
max_selected_features = 200
sel = SelectFromModel(model_lgb, max_features = max_selected_features, threshold=0.005, prefit=True)
 feature_idx = sel.get_support()
selected_features_lgb = X.columns[feature_idx]
seed = 43
test_size = 0.3
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=seed )<predict_on_test>
params = {
"metric": 'rmse',
"verbosity": -1
}
xtrain, xvalid, ytrain, yvalid = train_test_split(X[selected_features_lgb], y, random_state=42, test_size=0.3)
 predictions_test_lgb_fs = np.zeros(len(test_df))
num_fold = 0
oof_rmse = 0
num_of_splits = kfold_splits
folds = KFold(n_splits=num_of_splits, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Fold:", num_fold)
 print()
model_lgb_fs = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model_lgb_fs.fit(xtrain_stra, ytrain_stra,
eval_set=[(xtrain_stra, ytrain_stra),(xvalid_stra, yvalid_stra)], eval_metric='rmse',
verbose=100, early_stopping_rounds=1000)
 predictions_valid = model_lgb_fs.predict(xvalid)
 rmse_valid = np.sqrt(mean_squared_error(yvalid, predictions_valid))
print("Fold" ,num_fold, "xvalid rmse:", rmse_valid)
 num_fold = num_fold + 1
oof_rmse += rmse_valid
predictions_test_lgb_fs += model_lgb_fs.predict(test_df[xtrain.columns])/num_of_splits
predictions_test_lgb_fs = np.expm1(predictions_test_lgb_fs)
 print()
print(predictions_test_lgb_fs)
 print("OOF Out-of-fold rmse:", oof_rmse/num_of_splits)
 f, ax = plt.subplots(2, figsize=(12,7))
sns.set(rc={'figure.figsize':(9,14)})
 sns.distplot(train_df.revenue, ax=ax[0])
 ax[0].set_title("Train Set Revenue Histogram")
 sns.distplot(predictions_test_lgb_fs, ax=ax[1])
 ax[1].set_title("Test Set Revenue Prediction Histogram")
 f.tight_layout()
train_model5 = model3.fit(X_train, y_train)
 pred5 = train_model5.predict(X_test)
 print("Accuracy for model 5: %.2f" %(accuracy_score(y_test, pred5)* 100))<compute_test_metric>
result_lgb = bayesian_tuning_lgb(xtrain, ytrain )
print('Model 5 XGboost Report %r' %(classification_report(y_test, pred5)) )<train_model>
params = {
'num_leaves': result_lgbm.best_params_["num_leaves"],
'min_data_in_leaf': result_lgbm.best_params_["min_data_in_leaf"],
'max_depth': result_lgbm.best_params_["max_depth"],
'learning_rate': result_lgbm.best_params_["learning_rate"],
"boosting": "gbdt",
"feature_fraction": result_lgbm.best_params_["feature_fraction"],
"bagging_freq": 1,
"bagging_fraction": result_lgbm.best_params_["bagging_fraction"],
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": result_lgbm.best_params_["lambda_l1"],
"lambda_l2": result_lgbm.best_params_["lambda_l2"],
"verbosity": -1
}
xtrain, xvalid, ytrain, yvalid = train_test_split(X[selected_features_lgb], y, random_state=42, test_size=0.3)
 predictions_test_lgb_fs_tuned = np.zeros(len(test_df))
num_fold = 0
oof_rmse = 0
num_of_splits = kfold_splits
folds = KFold(n_splits=num_of_splits, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Fold:", num_fold)
 print()
model_lgb_fs_tuned = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 model_lgb_fs_tuned.fit(xtrain_stra, ytrain_stra,
eval_set=[(xtrain_stra, ytrain_stra),(xvalid_stra, yvalid_stra)], eval_metric='rmse',
verbose=100, early_stopping_rounds=1000)
 predictions_valid = model_lgb_fs_tuned.predict(xvalid)
 rmse_valid = np.sqrt(mean_squared_error(yvalid, predictions_valid))
print("Fold" ,num_fold, "xvalid rmse:", rmse_valid)
 num_fold = num_fold + 1
oof_rmse += rmse_valid
predictions_test_lgb_fs_tuned += model_lgb_fs_tuned.predict(test_df[xtrain.columns])/num_of_splits
predictions_test_lgb_fs_tuned = np.expm1(predictions_test_lgb_fs_tuned)
 print()
print(predictions_test_lgb_fs_tuned)
 print("OOF Out-of-fold rmse:", oof_rmse/num_of_splits)
 f, ax = plt.subplots(2, figsize=(12,7))
sns.set(rc={'figure.figsize':(9,14)})
 sns.distplot(train_df.revenue, ax=ax[0])
 ax[0].set_title("Train Set Revenue Histogram")
 sns.distplot(predictions_test_lgb_fs_tuned, ax=ax[1])
 ax[1].set_title("Test Set Revenue Prediction Histogram")
 f.tight_layout()
train_model6 = train_model5.fit(X, y)
<count_unique_values>
predictions_test_xgb_extra_trees =(0.5 * predictions_test_xgb)+(0.5 * predictions_extra_trees_test)
 predictions_test_xgb_random_forest =(0.5 * predictions_test_xgb)+(0.5 * predictions_random_forest_test)
 predictions_test_extra_trees_random_forest =(0.5 * predictions_extra_trees_test)+(0.5 * predictions_random_forest_test)
 predictions_test_tuned_xgb_extra_trees =(0.5 * predictions_test_xgb_tuned)+(0.5 * predictions_extra_trees_tuned_test)
 predictions_test_tuned_xgb_random_forest =(0.5 * predictions_test_xgb_tuned)+(0.5 * predictions_random_forest_tuned_test)
 predictions_test_tuned_extra_trees_random_forest =(0.5 * predictions_extra_trees_tuned_test)+(0.5 * predictions_random_forest_tuned_test)
 predictions_test_xgb_fs_extra_trees =(0.5 * predictions_test_xgb_fs)+(0.5 * predictions_extra_trees_test)
 predictions_test_xgb_fs_tuned_extra_trees =(0.5 * predictions_test_xgb_fs_tuned)+(0.5 * predictions_extra_trees_test)
 predictions_test_baseline_xgb_tuned_extra_trees =(0.5 * predictions_test_xgb)+(0.5 * predictions_extra_trees_tuned_test)
 predictions_test_lgb_xgb =(0.5 * predictions_test_lgb)+(0.5 * predictions_test_xgb)
 predictions_test_tuned_lgb_xgb =(0.5 * predictions_test_lgb_tuned)+(0.5 * predictions_test_xgb_tuned)
 predictions_test_lgb_extra_trees =(0.5 * predictions_test_lgb)+(0.5 * predictions_extra_trees_test)
 predictions_test_lgb_random_forest =(0.5 * predictions_test_lgb)+(0.5 * predictions_random_forest_test)
 predictions_test_tuned_lgb_extra_trees =(0.5 * predictions_test_lgb_tuned)+(0.5 * predictions_extra_trees_tuned_test)
 predictions_test_tuned_lgb_random_forest =(0.5 * predictions_test_lgb_tuned)+(0.5 * predictions_random_forest_tuned_test)
 predictions_test_xgb_fs_lgb =(0.5 * predictions_test_xgb_fs)+(0.5 * predictions_test_lgb_fs)
 predictions_test_xgb_fs_tuned_lgb =(0.5 * predictions_test_xgb_fs_tuned)+(0.5 * predictions_test_lgb_fs_tuned)
 predictions_test_baseline_lgb_tuned_extra_trees =(0.5 * predictions_test_lgb)+(0.5 * predictions_extra_trees_tuned_test )
<choose_model_class><EOS> 
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_xgb
submission.to_csv('clf_xgb_baseline.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_xgb_tuned
submission.to_csv('clf_xgb_tuned.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_lgb
submission.to_csv('clf_lgb_baseline.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_lgb_tuned
submission.to_csv('clf_lgb_tuned.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_extra_trees_test
submission.to_csv('clf_extra_trees_baseline.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_extra_trees_tuned_test
submission.to_csv('clf_extra_trees_tuned.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_xgb_fs
submission.to_csv('clf_xgb_fs_baseline.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_xgb_fs_tuned
submission.to_csv('clf_xgb_fs_tuned.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_lgb_fs
submission.to_csv('clf_lgb_fs_baseline.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_lgb_fs_tuned
submission.to_csv('clf_lgb_fs_tuned.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_xgb_extra_trees
submission.to_csv('blend_xgb_extra_trees_baselines.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_xgb_random_forest
submission.to_csv('blend_xgb_random_forest_baselines.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_extra_trees_random_forest
submission.to_csv('blend_extra_trees_random_forest_baselines.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_tuned_xgb_extra_trees
submission.to_csv('blend_xgb_extra_trees_tuned.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_tuned_xgb_random_forest
submission.to_csv('blend_xgb_random_forest_tuned.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_tuned_extra_trees_random_forest
submission.to_csv('blend_extra_trees_random_forest_tuned.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_baseline_xgb_tuned_extra_trees
submission.to_csv('blend_baseline_xgb_tuned_extra_trees.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_xgb_fs_extra_trees
submission.to_csv('blend_xgb_fs_extra_trees.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_xgb_fs_tuned_extra_trees
submission.to_csv('blend_xgb_fs_tuned_extra_trees.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_lgb_xgb
submission.to_csv('blend_lgb_xgb.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_tuned_lgb_xgb
submission.to_csv('blend_tuned_lgb_xgb.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_lgb_extra_trees
submission.to_csv('blend_tuned_lgb_extra_trees.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_lgb_extra_trees
submission.to_csv('blend_tuned_lgb_extra_trees.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_lgb_random_forest
submission.to_csv('blend_tuned_lgb_random_forest.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_tuned_lgb_extra_trees
submission.to_csv('blend_tuned_lgb_extra_trees.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_tuned_lgb_random_forest
submission.to_csv('blend_tuned_lgb_random_forest.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_baseline_lgb_tuned_extra_trees
submission.to_csv('blend_baseline_lgb_tuned_extra_trees.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_xgb_fs_lgb
submission.to_csv('blend_xgb_fs_lgb.csv', index=False)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['revenue'] = predictions_test_xgb_fs_tuned_lgb
submission.to_csv('blend_xgb_fs_tuned_lgb.csv', index=False)