Kaggle competition: Titanic - Machine Learning from Disaster
forest.fit(X_train, y_train)
 importance = forest.feature_importances_
indices = np.argsort(importance)[::-1]
for f in range(X_train.shape[1]):
print("%2d)%-*s %f" %(f+1, 30, X_train.columns[indices[f]], importance[indices[f]]))<find_best_model_class>
%matplotlib inline
%%time
scores = []
for i in range(50,len(indices),5):
forest = RandomForestRegressor()
sel_features = X_train.columns[indices[:i]]
X_tra, X_val, y_tra, y_val = train_test_split(X_train[sel_features], y_train, test_size=0.2, random_state=0)
 forest.fit(X_tra, y_tra)
 y_pred = forest.predict(X_val)
 score = r2_score(y_true=y_val, y_pred=y_pred)
 scores.append(score)
 plt.figure(figsize=(20,6))
plt.plot(range(50,len(indices),5), scores)
 plt.xlabel("Number of feature")
 plt.ylabel("Accuracy score" )<features_selection>
titanic_raw_train = pd.read_csv('.. //input/titanic/train.csv')
 titanic_raw_test = pd.read_csv('.. //input/titanic/test.csv')
 titanic_raw_train.info()
titanic_raw_test.info()
sel_features_forest = X_train.columns[indices[:130]]<compute_train_metric>
train = titanic_raw_train.copy()
test = titanic_raw_test.copy()
train_test = pd.concat([train,test] )
%%time
forest = RandomForestRegressor(n_estimators=100, max_depth=40, max_leaf_nodes=100, min_samples_leaf=1, min_samples_split=0.00001, random_state=10)
 print("Cross validation score")
 cr_score = cross_val_score(forest, X_train[sel_features_forest], y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<train_on_grid>
train[['Survived','Pclass']].groupby('Pclass' ).mean()
%%time
xgbr = xgb.XGBRegressor(random_state=10)
 opt_xgb = BayesSearchCV(
 xgbr,
{"learning_rate":(0.001,0.99),
"max_depth":(1,30),
"subsample":(0.1,0.99),
"colsample_bytree":(0.01,0.99)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_xgb = opt_xgb.fit(X_train, y_train)
 print("Best score:{}".format(bs_xgb.best_score_))
print("Best params:{}".format(bs_xgb.best_params_))<compute_train_metric>
train_test['Title']=train_test.Name.str.extract('([A-Za-z]+)\.' )
%%time
xgbr = xgb.XGBRegressor(learning_rate=0.1350, max_depth=3, subsample=0.7356, colsample_bytree=0.7682, random_state=10)
 print("Cross validation score")
 cr_score = cross_val_score(xgbr, X_train, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<define_search_model>
train_test.loc[train_test['Title'] == 'Mrs','Is_Married'] = 1
%%time
lgbm = lgb.LGBMRegressor()
opt_lgbm = BayesSearchCV(
 lgbm,
{"num_leaves":(20,100),
"n_estimators":(10,100),
"learning_rate":(0.001,1),
"max_depth":(1,30),
"min_split_gain":(0,0.5),
"min_child_weight":(0.001,0.1),
"min_child_samples":(5,50),
"subsample":(0.1,0.99),
"colsample_bytree":(0.01,0.99)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_lgbm = opt_lgbm.fit(X_train, y_train)
 print("Best score:{}".format(bs_lgbm.best_score_))
print("Best params:{}".format(bs_lgbm.best_params_))<compute_train_metric>
train_test['Family_name'] = train_test.Name.str.extract('(\w+),', expand=False)
 train_test.Family_name
%%time
lgbm = lgb.LGBMRegressor(learning_rate=0.2660, max_depth=26, colsample_bytree=0.6411, min_child_samples=36, min_child_weight=0.09835, min_split_gain=0, n_estimators=100, num_leaves=25, subsample=0.1146)
 print("Cross validation score")
 cr_score = cross_val_score(lgbm, X_train, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<train_model>
m = train_test[['Family_name', 'Survived']].groupby('Family_name' ).mean()
c = train_test[['Family_name', 'PassengerId']].groupby('Family_name' ).count()
m = m.rename(columns={'Survived': 'Family_survived'})
 c = c.rename(columns={'PassengerId': 'FamilyMemberCount'})
 m = m.where(m.join(c ).FamilyMemberCount > 1, other=-1 ).fillna(-1 ).join(c)
 m.Family_survived = m.Family_survived.astype('int64')
 train_test = train_test.join(m, on='Family_name' )
lgbm.fit(X_train, y_train)
 importance = lgbm.feature_importances_
indices = np.argsort(importance)[::-1]
for f in range(X_train.shape[1]):
print("%2d)%-*s %f" %(f+1, 30, X_train.columns[indices[f]], importance[indices[f]]))<find_best_model_class>
train_test.Age.isnull().sum()
%%time
scores = []
for i in range(50,len(indices),5):
lgbm = lgb.LGBMRegressor()
sel_features = X_train.columns[indices[:i]]
X_tra, X_val, y_tra, y_val = train_test_split(X_train[sel_features], y_train, test_size=0.2, random_state=0)
 lgbm.fit(X_tra, y_tra)
 y_pred = lgbm.predict(X_val)
 score = r2_score(y_true=y_val, y_pred=y_pred)
 scores.append(score)
 plt.figure(figsize=(20,6))
plt.plot(range(50,len(indices),5), scores)
 plt.xlabel("Number of feature")
 plt.ylabel("Accuracy score" )<features_selection>
train_test[['Age','Title']].groupby('Title' ).mean()
sel_features_lgbm = X_train.columns[indices[:100]]<compute_train_metric>
train_test.loc[(train_test.Age.isnull())&(train_test.Title=='Mr'),'Age']=32
train_test.loc[(train_test.Age.isnull())&(train_test.Title=='Mrs'),'Age']=37
train_test.loc[(train_test.Age.isnull())&(train_test.Title=='Master'),'Age']=5
train_test.loc[(train_test.Age.isnull())&(train_test.Title=='Miss'),'Age']=22
train_test.loc[(train_test.Age.isnull())&(train_test.Title=='Rare'),'Age']=45
%%time
lgbm = lgb.LGBMRegressor(learning_rate=0.2660, max_depth=26, colsample_bytree=0.6411, min_child_samples=36, min_child_weight=0.09835, min_split_gain=0, n_estimators=100, num_leaves=25, subsample=0.1146)
 print("Cross validation score")
 cr_score = cross_val_score(lgbm, X_train[sel_features_lgbm], y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<normalization>
train_test['Age'] = train_test['Age'].astype('int64' )
sc = StandardScaler()
sc.fit(X_train)
 X_train_std = sc.transform(X_train)
 X_test_std = sc.transform(X_test )<train_on_grid>
train[['Survived','Sex']].groupby('Sex' ).mean()
ridge = Ridge()
opt_r = BayesSearchCV(
 ridge,
{"alpha":(0.0001,1000)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_r = opt_r.fit(X_train_std, y_train)
 print("Best score:{}".format(bs_r.best_score_))
print("Best params:{}".format(bs_r.best_params_))<compute_train_metric>
train_test.Fare.isnull().sum()
ridge = Ridge(alpha=270.2)
 print("Cross validation score")
 cr_score = cross_val_score(ridge, X_train_std, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<train_on_grid>
train_test.Fare.fillna(test.Fare.median() , inplace=True )
lasso = Lasso()
opt_l = BayesSearchCV(
 lasso,
{"alpha":(0.00001,1)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_l = opt_l.fit(X_train_std, y_train)
 print("Best score:{}".format(bs_l.best_score_))
print("Best params:{}".format(bs_l.best_params_))<compute_train_metric>
train_test['Fare_log'] = np.log1p(train_test.Fare)
 plot_skew(train_test.Fare_log )
lasso = Lasso(alpha=0.00001)
 print("Cross validation score")
 cr_score = cross_val_score(lasso, X_train_std, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<train_on_grid>
train_test['Ticket_Frequency'] = train_test.groupby('Ticket')['Ticket'].transform('count')
 train_test.Ticket_Frequency.astype('int64' )
elas = ElasticNet()
opt_e = BayesSearchCV(
 elas,
{"alpha":(0.00001,1)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_e = opt_e.fit(X_train_std, y_train)
 print("Best score:{}".format(bs_e.best_score_))
print("Best params:{}".format(bs_e.best_params_))<compute_train_metric>
train_test.Cabin.isnull().sum()
elas = ElasticNet(alpha=0.00365)
 print("Cross validation score")
 cr_score = cross_val_score(elas, X_train_std, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<choose_model_class>
train_test['Cabin_code'] = train_test.Cabin.str.get(0 ).fillna('Z')
 train_test[['Survived','Cabin_code']].groupby('Cabin_code' ).mean()
%%time
estimators = [("tr", DecisionTreeRegressor(max_depth=9, max_leaf_nodes=67, random_state=10)) , 
("rf", RandomForestRegressor(n_estimators=100, max_depth=40, max_leaf_nodes=100, min_samples_leaf=1, min_samples_split=0.00001, random_state=10)) , 
("xg", xgb.XGBRegressor(learning_rate=0.1350, max_depth=3, subsample=0.7356, colsample_bytree=0.7682, random_state=10)) , 
("lg", lgb.LGBMRegressor(learning_rate=0.2660, max_depth=26, colsample_bytree=0.6411, min_child_samples=36, min_child_weight=0.09835, min_split_gain=0, n_estimators=100, num_leaves=25, subsample=0.1146)) , 
("ri", Ridge(alpha=5.91)) , 
("la", Lasso(alpha=0.00001)) , 
("el", ElasticNet(alpha=0.00069)) ]
sreg = StackingRegressor(estimators=estimators)
 sreg.fit(X_train, y_train)
 print("Score:{}".format(sreg.score(X_train, y_train)))
 print("Params:{}".format(sreg.get_params))<compute_train_metric>
train.Embarked.isnull().sum()
%%time
print("Cross validation score")
 cr_score = cross_val_score(sreg, X_train, y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<predict_on_test>
train[['Embarked','Survived']].groupby('Embarked' ).mean()
ps_test = sreg.predict(X_test)
 X_test["y:pseudo"] = ps_test<concatenate>
train_test['Family_size']=train_test.SibSp+train_test.Parch
train_test['Is_alone']=1
train_test.loc[train_test['Family_size'] > 1,'Is_alone'] = 0
train_test['Family_size'].astype('int64')
 train_test['Is_alone'].astype('int64' )
Ps_X_train = X_test.sample(int(len(X_train)/2))
Ps_X_train.reset_index(drop=True, inplace=True)
 Ps_y_train = Ps_X_train["y:pseudo"]
Ps_X_train = Ps_X_train.drop("y:pseudo", axis=1)
 n_X_train = pd.concat([X_train, Ps_X_train] ).reset_index(drop=True)
 n_y_train = pd.concat([y_train, Ps_y_train] ).reset_index(drop=True )<train_on_grid>
%%time
tree = DecisionTreeRegressor(random_state=10)
 opt_tree = BayesSearchCV(
 tree,
{"max_depth":(5,20),
"max_leaf_nodes":(30,100)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_tree = opt_tree.fit(n_X_train, n_y_train)
 print("Best score:{}".format(bs_tree.best_score_))
print("Best params:{}".format(bs_tree.best_params_))<compute_train_metric>
train_test['Age_band'] = pd.cut(train_test.Age, bins=10, precision=0 )
%%time
tree = DecisionTreeRegressor(max_depth=10, max_leaf_nodes=72, random_state=10)
 print("Cross validation score")
 cr_score = cross_val_score(tree, n_X_train, n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<train_on_grid>
train_test['Fare_band'] = pd.qcut(train_test.Fare_log, q=13, precision=2 )
%%time
forest = RandomForestRegressor(random_state=10)
 opt_forest = BayesSearchCV(
 forest,
{"max_depth":(5,40),
"max_leaf_nodes":(30,100),
"n_estimators":(50,100),
"min_samples_split":(0.00001,1.0),
"min_samples_leaf":(1,10)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_forest = opt_forest.fit(n_X_train[sel_features_forest], n_y_train)
 print("Best score:{}".format(bs_forest.best_score_))
print("Best params:{}".format(bs_forest.best_params_))<compute_train_metric>
MM_scaler = MinMaxScaler(feature_range=(0,1))
train_test[['Age_scaled']]=MM_scaler.fit_transform(train_test[['Age']])
 sns.distplot(a=train_test.Age_scaled )
%%time
forest = RandomForestRegressor(n_estimators=100, max_depth=40, max_leaf_nodes=100, min_samples_leaf=1, min_samples_split=0.00001, random_state=10)
 print("Cross validation score")
 cr_score = cross_val_score(forest, n_X_train[sel_features_forest], n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<train_on_grid>
St_scaler = StandardScaler()
train_test[['Fare_scaled']]=St_scaler.fit_transform(train_test[['Fare_log']])
 sns.distplot(a=train_test.Fare_scaled )
%%time
xgbr = xgb.XGBRegressor(random_state=10)
 opt_xgb = BayesSearchCV(
 xgbr,
{"learning_rate":(0.001,0.99),
"max_depth":(1,30),
"subsample":(0.1,0.99),
"colsample_bytree":(0.01,0.99)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_xgb = opt_xgb.fit(n_X_train, n_y_train)
 print("Best score:{}".format(bs_xgb.best_score_))
print("Best params:{}".format(bs_xgb.best_params_))<compute_train_metric>
le = OrdinalEncoder()
%%time
xgbr = xgb.XGBRegressor(learning_rate=0.07403, max_depth=211, subsample=0.7613, colsample_bytree=0.7985, random_state=10)
 print("Cross validation score")
 cr_score = cross_val_score(xgbr, n_X_train, n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<define_search_model>
train_test[['Age_code']] = le.fit_transform(train_test[['Age_band']])
 train_test[['Fare_code']] = le.fit_transform(train_test[['Fare_band']] )
%%time
lgbm = lgb.LGBMRegressor()
opt_lgbm = BayesSearchCV(
 lgbm,
{"num_leaves":(20,100),
"n_estimators":(10,100),
"learning_rate":(0.001,1),
"max_depth":(1,30),
"min_split_gain":(0,0.5),
"min_child_weight":(0.001,0.1),
"min_child_samples":(5,50),
"subsample":(0.1,0.99),
"colsample_bytree":(0.01,0.99)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_lgbm = opt_lgbm.fit(n_X_train[sel_features_lgbm], n_y_train)
 print("Best score:{}".format(bs_lgbm.best_score_))
print("Best params:{}".format(bs_lgbm.best_params_))<compute_train_metric>
for row in train_test:
train_test.loc[train_test['Family_size']==0, 'Family_type']=1
train_test.loc[(1<=train_test['Family_size'])&(train_test['Family_size']<=3), 'Family_type']=2
train_test.loc[(3<=train_test['Family_size'])&(train_test['Family_size']<=6), 'Family_type']=3
train_test.loc[7<=train_test['Family_size'], 'Family_type']=4
train_test.Family_type = train_test.Family_type.astype('int64' )
%%time
lgbm = lgb.LGBMRegressor(learning_rate=0.1297, max_depth=20, colsample_bytree=0.9295, min_child_samples=43, min_child_weight=0.0484, min_split_gain=0, n_estimators=77, num_leaves=22, subsample=0.1)
 print("Cross validation score")
 cr_score = cross_val_score(lgbm, n_X_train[sel_features_lgbm], n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<normalization>
n_X_train_std = sc.transform(n_X_train )<train_on_grid>
train_test.Sex.replace(['female', 'male'], [1,0],inplace=True )
%%time
ridge = Ridge()
opt_r = BayesSearchCV(
 ridge,
{"alpha":(0.0001,1000)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_r = opt_r.fit(n_X_train_std, n_y_train)
 print("Best score:{}".format(bs_r.best_score_))
print("Best params:{}".format(bs_r.best_params_))<compute_train_metric>
X = pd.get_dummies(train_test[['Pclass','Sex','Title','Family_type',
'Age_code','Fare_code','Embarked',
'Cabin_code','Ticket_Code_Remap','Family_survived']])[:891]
X_test = pd.get_dummies(train_test[['Pclass','Sex','Title','Family_type',
'Age_code','Fare_code','Embarked',
'Cabin_code','Ticket_Code_Remap','Family_survived']])[891:]
Y = train.Survived
X.shape
%%time
ridge = Ridge(alpha=70.98)
 print("Cross validation score")
 cr_score = cross_val_score(ridge, n_X_train_std, n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<train_on_grid>
CV = StratifiedKFold(n_splits=5, shuffle=True,random_state=42)
 n_trials = 500
%%time
lasso = Lasso()
opt_l = BayesSearchCV(
 lasso,
{"alpha":(0.00001,1)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_l = opt_l.fit(n_X_train_std, n_y_train)
 print("Best score:{}".format(bs_l.best_score_))
print("Best params:{}".format(bs_l.best_params_))<compute_train_metric>
%%time
lasso = Lasso(alpha=0.00012)
 print("Cross validation score")
 cr_score = cross_val_score(lasso, n_X_train_std, n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<train_on_grid>
%%time
elas = ElasticNet()
opt_e = BayesSearchCV(
 elas,
{"alpha":(0.00001,1)},
n_iter = 50,
cv = 5,
n_jobs = -1)
 bs_e = opt_e.fit(n_X_train_std, n_y_train)
 print("Best score:{}".format(bs_e.best_score_))
print("Best params:{}".format(bs_e.best_params_))<compute_train_metric>
LR_param = {'penalty': 'l2', 'C': 109.16587461586346}
LR_best= LogisticRegression(**LR_param )
%%time
elas = ElasticNet(alpha=0.00011)
 print("Cross validation score")
 cr_score = cross_val_score(elas, n_X_train_std, n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<choose_model_class>
DT_param ={'max_depth': 12, 'min_samples_split': 60, 'min_samples_leaf': 20}
DT_best= DecisionTreeClassifier(**DT_param )
%%time
estimators = [("tr", DecisionTreeRegressor(max_depth=10, max_leaf_nodes=72, random_state=10)) , 
("rf", RandomForestRegressor(n_estimators=100, max_depth=40, max_leaf_nodes=100, min_samples_leaf=1, min_samples_split=0.00001, random_state=10)) , 
("xg", xgb.XGBRegressor(learning_rate=0.07403, max_depth=211, subsample=0.7613, colsample_bytree=0.7985, random_state=10)) , 
("lg", lgb.LGBMRegressor(learning_rate=0.1297, max_depth=20, colsample_bytree=0.9295, min_child_samples=43, min_child_weight=0.0484, min_split_gain=0, n_estimators=77, num_leaves=22, subsample=0.1)) , 
("ri", Ridge(alpha=0.0001)) , 
("la", Lasso(alpha=0.00001)) , 
("el", ElasticNet(alpha=0.00001)) ]
sreg = StackingRegressor(estimators=estimators)
 sreg.fit(n_X_train, n_y_train)
 print("Score:{}".format(sreg.score(n_X_train, n_y_train)))
 print("Params:{}".format(sreg.get_params))<compute_train_metric>
KNN_param ={'n_neighbors': 5, 'weights': 'uniform', 'p': 1}
KNN_best= KNeighborsClassifier(**KNN_param )
print("Cross validation score")
 cr_score = cross_val_score(sreg, n_X_train, n_y_train, cv=10)
 print(cr_score)
 print("Ave_score:{} ± {}".format(cr_score.mean() , cr_score.std()))<normalization>
X_test.drop("y:pseudo", axis=1, inplace=True)
 X_test_std = sc.transform(X_test )<predict_on_test>
y_pred_tree = 10**(bs_tree.predict(X_test))
y_pred_forest = 10**(bs_forest.predict(X_test[sel_features_forest]))
y_pred_xgb = 10**(bs_xgb.predict(X_test))
y_pred_lgbm = 10**(bs_lgbm.predict(X_test[sel_features_lgbm]))
y_pred_r = 10**(bs_r.predict(X_test_std))
y_pred_l = 10**(bs_l.predict(X_test_std))
y_pred_sreg = 10**(sreg.predict(X_test))<prepare_output>
submit_xgb = sample.copy()
submit_xgb["SalePrice"] = y_pred_xgb<prepare_output>
MLP_best= MLPClassifier(random_state=42,
solver='adam',
early_stopping=True,
activation='relu',
alpha= 0.0002746340910250398,
learning_rate='adaptive',
learning_rate_init=0.01,
batch_size=32,
hidden_layer_sizes=160 )
submit_lgbm = sample.copy()
submit_lgbm["SalePrice"] = y_pred_lgbm<prepare_output>
RF_param={'n_estimators': 50, 'max_depth': 5, 'min_samples_split': 40, 'min_samples_leaf': 20}
RF_best= RandomForestClassifier(random_state=42,
criterion='gini',
oob_score=True,
**RF_param )
submit_sreg = sample.copy()
submit_sreg["SalePrice"] = y_pred_sreg<prepare_output>
ET_params = {'n_estimators': 250, 'max_depth': 5, 'min_samples_split': 20, 'min_samples_leaf': 20}
ET_best = ExtraTreesClassifier(random_state=42,
criterion='gini',
n_jobs=-1,
**ET_params )
submit_ems = sample.copy()
submit_ems["SalePrice"] = y_pred_tree*0.05 + y_pred_forest*0.1 + y_pred_xgb*0.3 + y_pred_lgbm*0.2 + y_pred_r*0.05 + y_pred_l*0.05 + y_pred_sreg*0.25<save_to_csv>
SVMB_params = {'n_estimators': 300, 'base_estimator__C': 3.232901108594473, 'base_estimator__gamma': 0.07183110256410177}
SVMB_best = BaggingClassifier(base_estimator=SVC(kernel = 'rbf',probability=True,C=3.232901108594473,gamma=0.07183110256410177),
random_state=42,n_jobs=-1,n_estimators=350 )
submit_xgb.to_csv('xgb_submission.csv', index=False)
 submit_lgbm.to_csv('lgbm_submission.csv', index=False)
 submit_sreg.to_csv('stack_submission.csv', index=False)
 submit_ems.to_csv('ems_submission.csv', index=False)
 print("Your submission was successfully saved!" )<import_modules>
ADB_best= AdaBoostClassifier(base_estimator=DecisionTreeClassifier(criterion='gini',
max_depth=2),
learning_rate=0.03990900089241141 ,
n_estimators=160,
random_state=42 )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import norm<load_from_csv>
GDB_param = {'n_estimators': 420, 'learning_rate': 0.08199591231901683,
'max_depth': 3, 'min_samples_split': 140, 'min_samples_leaf': 20}
GDB_best= GradientBoostingClassifier(**GDB_param,
subsample=0.8,
n_iter_no_change=10,
random_state=42 )
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )<drop_column>
train_id = train['Id']
test_id = test['Id']
train = train.drop('Id', axis=1)
 test = test.drop('Id', axis=1 )<drop_column>
XGB_param = {'max_depth': 5, 'n_estimators': 500, 'booster': 'dart',
'min_child_weight': 45, 'learning_rate': 0.00028818174062883895,
'gamma': 0.0701754028803822, 'reg_alpha': 0.09673762960851098, 'reg_lambda': 0.020973617864068886,
'colsample_bytree': 0.6000000000000001, 'subsample': 1.0}
XGB_best= XGBClassifier(**XGB_param,
objective='binary:logistic',
n_jobs=-1,
random_state=42 )
to_drop = ['PoolQC', 'MiscFeature', 'Alley', 'FireplaceQu']
train = train.drop(to_drop, axis=1)
 test = test.drop(to_drop, axis=1 )<data_type_conversions>
votingC = VotingClassifier(estimators=[('XGB', XGB_best), 
('RF',RF_best), 
('ADB',ADB_best), 
('ET', ET_best), 
('GDB',GDB_best), 
('DT',DT_best)],
voting='hard', n_jobs=-1,verbose=True)
 votingC.fit(X,Y)
 cv_result = cross_val_score(votingC,X,Y, cv = 5,scoring = "accuracy")
 vot_acc = cv_result.mean()
vot_std = cv_result.std()
performance_df.loc['Voting'] = {'CV_Mean':vot_acc, 'Std':vot_std}
performance_df.sort_values(by=['CV_Mean'],ascending=False )
LOG_FEATURES = ['LotFrontage', 'LotArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces', 'GarageCars', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'PoolArea']
def engineer_features(data):
data['MSSubClass'] = data['MSSubClass'].astype(str)
 data['YrSold'] = data['YrSold'].astype(str)
 data['MoSold'] = data['MoSold'].astype(str)
 data['haspool'] = data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 data['has2ndfloor'] = data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 data['hasgarage'] = data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 data['hasbsmt'] = data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 data['hasfireplace'] = data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 for feature in LOG_FEATURES:
data[feature] = np.log1p(data[feature])
 return data<normalization>
stacking = StackingClassifier(estimators=[('XGB', XGB_best), 
('RF',RF_best), 
('ADB',ADB_best), 
('ET', ET_best), 
('GDB',GDB_best), 
('DT',DT_best)],
final_estimator=LogisticRegression() ,
cv=5,
n_jobs=-1)
 cv_result = cross_val_score(stacking,X,Y, cv = 5,scoring = "accuracy")
 stk_acc = cv_result.mean()
stk_std = cv_result.std()
performance_df.loc['Stacking'] = {'CV_Mean':stk_acc, 'Std':stk_std}
performance_df.sort_values(by=['CV_Mean'],ascending=False )
train = engineer_features(train)
 test = engineer_features(test )<import_modules>
X_tr, X_te, Y_tr, Y_te = train_test_split(X,Y)
 clf = RF_best.fit(X_tr,Y_tr)
 mis_df = X_te[np.logical_xor(Y_te,clf.predict(X_te)) ]
mis_df=train.loc[mis_df.index]
mis_df.describe()
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV, cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer, make_column_selector, TransformedTargetRegressor
from sklearn.preprocessing import OneHotEncoder, RobustScaler, OrdinalEncoder, StandardScaler
from sklearn.linear_model import Ridge, Lasso, RidgeCV
from sklearn.neighbors import KNeighborsRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestRegressor, StackingRegressor
from sklearn.metrics import mean_squared_error, mean_squared_log_error, make_scorer
import xgboost
import lightgbm as lgb<prepare_x_and_y>
titanic_raw_train.loc[mis_df.index]
X_train, y_train = train.drop('SalePrice', axis=1), train['SalePrice']<define_variables>
sub_model = RF_best
categorical = [i for i in X_train.columns if train.dtypes[i] == 'object']
numerical = [i for i in X_train.columns if train.dtypes[i] != 'object']<categorify>
clf = RF_best.fit(X,Y)
 sub = clf.predict(X_test)
 sub_pd = pd.DataFrame({'PassengerId':titanic_raw_test.PassengerId,'Survived':sub})
 sub_pd.to_csv('submit.csv' ,index=False )