Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<feature_engineering>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from numpy import nan
rfc = RandomForestRegressor(n_estimators=100)
 rfc = rfc.fit(Xtrain, Ytrain)
 Ypredict = rfc.predict(Xtest)
 all_data.loc[all_data.loc[:,"Age"].isnull() ,"Age"] = Ypredict
all_data["Age"] = all_data["Age"].apply(lambda x:round(x))
nameList = ["Sex","Pclass","Fare","Title","Age","FamilyLabel","TicketLabel","Embarked"]
df_data = all_data.loc[:,nameList]
df_data = SimpleImputer(missing_values=np.nan,strategy='constant',fill_value=0 ).fit_transform(df_data)
 df_data = pd.DataFrame(df_data)
 df_data.info()<prepare_x_and_y>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
Y = all_data.loc[:,"Survived"]
Y_predict = Y[Y.isnull() ]
Y_train = Y[Y.notnull() ]
X_train = df_data.iloc[Y_train.index,:]
X_predict = df_data.iloc[Y_predict.index,:]<compute_train_metric>
y_train = train['SalePrice']
train.drop(['SalePrice'],axis=1,inplace=True)
 features = pd.concat([train, test])
 features.shape
rfc = RandomForestClassifier(n_estimators=100,random_state=90)
 score_pre = cross_val_score(rfc,X_train,Y_train,cv=10 ).mean()
score_pre<find_best_params>
drop_columns = ['Id','Alley','FireplaceQu','PoolQC','Fence','MiscFeature','GarageYrBlt']
features.drop(drop_columns,axis=1,inplace=True )
score_list=[]
for i in range(0,200,10):
rfc = RandomForestClassifier(n_estimators=i+1,random_state=90,n_jobs=-1)
 score = cross_val_score(rfc,X_train,Y_train,cv=10 ).mean()
score_list.append(score)
 print(max(score_list),(score_list.index(max(score_list)) *10)+1 )<find_best_params>
numeric_dtypes = ['int64', 'float64']
for column in features.columns :
if features[column].dtype in numeric_dtypes :
features[column] = features[column].fillna(features[column].mean())
 else :
features[column] = features[column].fillna(features[column].mode() [0])
score_list=[]
for i in range(25,40):
rfc = RandomForestClassifier(n_estimators=i, random_state=90, n_jobs=-1)
 score = cross_val_score(rfc,X_train,Y_train,cv=10 ).mean()
score_list.append(score)
 print(max(score_list),([*range(25,40)][score_list.index(max(score_list)) ]))<train_on_grid>
for column in features.columns :
if features[column].dtype == object :
df_dummies = pd.get_dummies(features[column],drop_first=True,prefix=column)
 features.drop([column],axis=1,inplace=True)
 features = pd.concat([features,df_dummies],axis=1)
 features.head()
param_grid = {"max_depth":np.arange(1,20,1)}
rfc = RandomForestClassifier(n_estimators=30, random_state=90)
 GS = GridSearchCV(rfc,param_grid,cv=10)
 GS.fit(X_train,Y_train)
 GS.best_params_<find_best_score>
x_train = features.iloc[:len(y_train), :]
x_test = features.iloc[len(x_train):, :]
GS.best_score_<train_on_grid>
def bi_directional_elimination(x, y):
Sig_Level = 0.05;
features = x.columns.tolist()
selected_features = []
while(len(features)> 0):
remain_features = list(set(features)- set(selected_features))
pval = pd.Series(index=remain_features, dtype='float64')
 for column in remain_features :
model = sm.OLS(y, sm.add_constant(x[selected_features + [column]])).fit()
pval[column] = model.pvalues[column]
min_pval = pval.min()
if(min_pval < Sig_Level):
selected_features.append(pval.idxmin())
 while(len(selected_features)> 0):
selected_features_with_constant = sm.add_constant(x[selected_features])
 p_values = sm.OLS(y, selected_features_with_constant ).fit().pvalues[1:]
max_pval = p_values.max()
if(max_pval >= Sig_Level):
removed_feature = p_values.idxmax()
selected_features.remove(removed_feature)
 else :
break
else :
break
return selected_features
param_grid={'min_samples_leaf':np.arange(1,1+10,1)}
rfc = RandomForestClassifier(
 n_estimators=30,
random_state=90,
max_depth=8 
)
 GS = GridSearchCV(rfc,param_grid,cv=10)
 GS.fit(X_train,Y_train)
 GS.best_params_<find_best_score>
selected_features = bi_directional_elimination(x_train, y_train )
GS.best_score_<train_on_grid>
x_train = x_train[selected_features].copy()
x_test = x_test[selected_features].copy()
param_grid={'min_samples_split':np.arange(2, 2+20, 1)}
rfc = RandomForestClassifier(
 n_estimators=30,
random_state=90,
max_depth=8,
min_samples_leaf=2 
)
 GS = GridSearchCV(rfc,param_grid,cv=10)
 GS.fit(X_train,Y_train)
 GS.best_params_<find_best_score>
regressor = xgboost.XGBRegressor()
booster=['gbtree','gblinear']
base_score=[0.25,0.5,0.75,1]
n_estimators = [100, 500, 900, 1100, 1500]
max_depth = [2, 3, 5, 10, 15]
learning_rate=[0.05,0.1,0.15,0.20]
min_child_weight=[1,2,3,4]
hyperparameter_grid = {
'n_estimators': n_estimators,
'max_depth':max_depth,
'learning_rate':learning_rate,
'min_child_weight':min_child_weight,
'booster':booster,
'base_score':base_score
}
random_cv = RandomizedSearchCV(estimator=regressor,
param_distributions=hyperparameter_grid,
cv=5, n_iter=50,
scoring = 'neg_mean_absolute_error',n_jobs = 4,
verbose = 5,
return_train_score = True,
random_state=42)
 random_cv.fit(x_train,y_train )
GS.best_score_<compute_train_metric>
random_cv.best_estimator_
rfc = RandomForestClassifier(
 n_estimators=30,
random_state=90,
max_depth=8,
min_samples_leaf=2,
min_samples_split=11 
)
 score = cross_val_score(rfc,X_train,Y_train,cv=10 ).mean()
score<predict_on_test>
regressor = xgboost.XGBRegressor(base_score=0.25, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.1, max_delta_step=0, max_depth=2,
min_child_weight=1, missing=nan, monotone_constraints='() ',
n_estimators=900, n_jobs=8, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,
tree_method='exact', validate_parameters=1, verbosity=None )
rfc.fit(X_train, Y_train)
 predictions = rfc.predict(X_predict ).astype(int )<save_to_csv>
regressor.fit(x_train,y_train )
output = pd.DataFrame({'PassengerId': test.PassengerId, 'Survived': predictions})
 output.to_csv('submission2.csv', index=False)
 print("Your submission was successfully saved!" )<set_options>
y_pred = regressor.predict(x_test )
<load_from_csv><EOS> 
df_pred = pd.DataFrame(y_pred)
 df_sub = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv')
 df_sub = pd.concat([df_sub['Id'],df_pred],axis=1)
 df_sub.columns = ['Id','SalePrice']
df_sub.to_csv('sample_submission.csv',index=False )