Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<feature_engineering>
%matplotlib inline
warnings.filterwarnings("ignore" )
df_test['CategoricalFare'] = 0
df_test.loc[df_test['Fare'] <= 7.55, 'CategoricalFare'] = 0
df_test.loc[(df_test['Fare'] > 7.55)&(df_test['Fare'] <= 7.854), 'CategoricalFare'] = 1
df_test.loc[(df_test['Fare'] > 7.854)&(df_test['Fare'] <= 8.05), 'CategoricalFare'] = 2
df_test.loc[(df_test['Fare'] > 8.05)&(df_test['Fare'] <= 10.5), 'CategoricalFare'] = 3
df_test.loc[(df_test['Fare'] > 10.5)&(df_test['Fare'] <= 14.454), 'CategoricalFare'] = 4
df_test.loc[(df_test['Fare'] > 14.454)&(df_test['Fare'] <= 21.679), 'CategoricalFare'] = 5
df_test.loc[(df_test['Fare'] > 21.679)&(df_test['Fare'] <= 27.0), 'CategoricalFare'] = 6
df_test.loc[(df_test['Fare'] > 27.0)&(df_test['Fare'] <= 39.688), 'CategoricalFare'] = 7
df_test.loc[(df_test['Fare'] > 39.688)&(df_test['Fare'] <= 77.958), 'CategoricalFare'] = 8
df_test.loc[df_test['Fare'] > 77.958, 'CategoricalFare'] = 9
df_test.head()<drop_column>
train=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
drop_elements = ['Name', 'Ticket', 'Cabin', 'Age', 'Fare', 'Parch', 'SibSp', 'IsAlone']
df_train = df_train.drop(drop_elements, axis = 1)
 df_test = df_test.drop(drop_elements, axis = 1 )<drop_column>
train_y=train['SalePrice']
train_y=np.log1p(train_y)
 train=train.drop('SalePrice',axis=1)
 df=pd.concat([train,test],axis=0 )
train = df_train.copy()
train = train.drop('PassengerId', axis=1)
 train.head()<drop_column>
for col in ['Alley','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','FireplaceQu',
'GarageType','GarageFinish','GarageQual','GarageCond','PoolQC','Fence','MiscFeature']:
df[col]=df[col].fillna(value='No' )
test = df_test.copy()
test = test.drop('PassengerId', axis=1)
 test.head()<import_modules>
tmp=df.astype(object)
 deg_=[]
for col in list(tmp.columns):
tol=tmp[col].value_counts()
if(tol.iloc[0]//tol.iloc[1]>30)&(sum(tol.iloc[1:])/tmp.shape[0]<=0.1):
print('degenerate column:',col,': with ratio',sum(tol.iloc[1:])/tmp.shape[0],'and',
tol.iloc[0]//tol.iloc[1])
 deg_.append(col )
from sklearn.preprocessing import StandardScaler<data_type_conversions>
df=df.drop(deg_,axis=1 )
X_train = train.drop('Survived', axis=1 ).astype(float)
 y_train = df_train['Survived']
X_test = test.astype(float )<normalization>
pd.DataFrame(df.isnull().sum() /df.shape[0] ).sort_values(0,ascending=False)[:20]
scaler = StandardScaler()
scaler.fit(X_train )<normalization>
num_=[]
for col in list(df.columns)[1:]:
if np.dtype(train[col])!=object:
num_.append(col)
 df_num=df[num_]
df_obj=df.drop(num_,axis=1 )
X_train = scaler.transform(X_train )<normalization>
high=list(pd.DataFrame(tmp['y'] ).sort_values('y',ascending=False ).index[1:11] )
X_test = scaler.transform(X_test )<import_modules>
df_num.loc[df_num.GarageYrBlt>2020,'GarageYrBlt']=np.nan
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neural_network import MLPClassifier<compute_train_metric>
df_num['y']=train_y
logreg = LogisticRegression(solver='lbfgs')
 logreg.fit(X_train, y_train)
 Y_pred = logreg.predict(X_test)
 acc_log = round(logreg.score(X_train, y_train)* 100, 3)
 acc_log<compute_train_metric>
df_num.loc[(df_num.OverallQual==4)&(df_num.y>12.4),'OverallQual']=np.nan
df_num.loc[(df_num.OverallQual==4)&(df_num.y<10.7),'OverallQual']=np.nan
df_num.loc[(df_num.OverallQual==3)&(df_num.y<10.7),'OverallQual']=np.nan
df_num.loc[(df_num.OverallQual==2)&(df_num.y<10.7),'OverallQual']=np.nan
df_num.loc[(df_num.OverallQual==10)&(df_num.y<12.3),'OverallQual']=np.nan
svc = SVC(gamma='scale')
 svc.fit(X_train, y_train)
 Y_pred = svc.predict(X_test)
 acc_svc = round(svc.score(X_train, y_train)* 100, 3)
 acc_svc<predict_on_test>
df_num.loc[(df_num.y<11),'GrLivArea']=np.nan
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, y_train)
 Y_pred = knn.predict(X_test)
 acc_knn = round(knn.score(X_train, y_train)* 100, 3)
 acc_knn<predict_on_test>
df_num.loc[(df_num.y<10.7),'GarageCars']=np.nan
gaussian = GaussianNB()
gaussian.fit(X_train, y_train)
 Y_pred = gaussian.predict(X_test)
 acc_gaussian = round(gaussian.score(X_train, y_train)* 100, 3)
 acc_gaussian<compute_train_metric>
df_num.loc[(df_num.y<10.7),'GarageArea']=np.nan
df_num.loc[(df_num.y<12.6)&(df_num.GarageArea>1200),'GarageArea']=np.nan
perceptron = Perceptron(max_iter=1000, tol=0.001)
 perceptron.fit(X_train, y_train)
 Y_pred = perceptron.predict(X_test)
 acc_perceptron = round(perceptron.score(X_train, y_train)* 100, 3)
 acc_perceptron<compute_train_metric>
df_num=df_num.drop('y',axis=1 )
linear_svc = LinearSVC(max_iter=10000)
 linear_svc.fit(X_train, y_train)
 Y_pred = linear_svc.predict(X_test)
 acc_linear_svc = round(linear_svc.score(X_train, y_train)* 100, 3)
 acc_linear_svc<compute_train_metric>
for col in list(df_num.columns):
if abs(skew(df_num[col])) >0.5:
df_num[col]=boxcox1p(df_num[col],boxcox_normmax(df_num[col]+1))
sgd = SGDClassifier(max_iter=10000, tol=0.001)
 sgd.fit(X_train, y_train)
 Y_pred = sgd.predict(X_test)
 acc_sgd = round(sgd.score(X_train, y_train)* 100, 3)
 acc_sgd<choose_model_class>
std=StandardScaler()
df_num=pd.DataFrame(std.fit_transform(df_num),columns=df_num.columns )
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, y_train)
 Y_pred = decision_tree.predict(X_test)
 acc_decision_tree = round(decision_tree.score(X_train, y_train)* 100, 3)
 acc_decision_tree<compute_train_metric>
pca=PCA(n_components=25)
 df_num=pd.DataFrame(pca.fit_transform(df_num),columns=['PC'+str(i)for i in range(1,26)] )
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, y_train)
 Y_pred = random_forest.predict(X_test)
 random_forest.score(X_train, y_train)
 acc_random_forest = round(random_forest.score(X_train, y_train)* 100, 3)
 acc_random_forest<train_model>
df_num['y']=train_y
df_num.loc[(df_num.PC1>7.5)&(df_num.y<12.5),'PC1']=np.nan
df_num=df_num.drop('y',axis=1 )
MLP_clf = MLPClassifier(activation='tanh', solver='lbfgs', alpha=1e-7, hidden_layer_sizes=(50,50), random_state=1, warm_start=True)
 MLP_clf.fit(X_train, y_train)
 Y_pred = MLP_clf.predict(X_test)
 acc_MLP_clf = round(MLP_clf.score(X_train, y_train)* 100, 3)
 acc_MLP_clf<choose_model_class>
imp=SimpleImputer(strategy='median')
 df_num=pd.DataFrame(imp.fit_transform(df_num),columns=df_num.columns )
adaboost = AdaBoostClassifier()
adaboost.fit(X_train, y_train)
 Y_pred = adaboost.predict(X_test)
 acc_adaboost = round(adaboost.score(X_train, y_train)* 100, 3)
 acc_adaboost<choose_model_class>
df_obj=df_obj.drop('Id',axis=1)
 pd.DataFrame(df_obj.isnull().sum() ).sort_values(0,ascending=False)[:10]
gboost = GradientBoostingClassifier()
gboost.fit(X_train, y_train)
 Y_pred = gboost.predict(X_test)
 acc_gboost = round(gboost.score(X_train, y_train)* 100, 3)
 acc_gboost<create_dataframe>
median_col=pd.DataFrame(df_obj.isnull().sum() ).sort_values(0,ascending=False ).index[1:7]
median_col
models = pd.DataFrame({
'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest', 'Naive Bayes', 'Perceptron',
'Stochastic Gradient Decent', 'Linear SVC',
'Decision Tree', 'MLPClassifier', 'AdaBoostClassifier', 'GradientBoostingClassifier'],
'Score': [acc_svc, acc_knn, acc_log,
acc_random_forest, acc_gaussian, acc_perceptron,
acc_sgd, acc_linear_svc, acc_decision_tree, acc_MLP_clf, acc_adaboost, acc_gboost]})
 models.sort_values(by='Score', ascending=False )<predict_on_test>
for col in list(median_col):
df_obj[col]=df_obj[col].fillna(df_obj[col].value_counts().index[0] )
Y_pred = random_forest.predict(X_test )<save_to_csv>
df_obj_MasVnrType=df_obj['MasVnrType']
df_obj_without=df_obj.drop('MasVnrType',axis=1 )
submission = pd.DataFrame({
"PassengerId": df_test["PassengerId"],
"Survived": Y_pred
})
 submission.to_csv('submission.csv', index=False )<load_from_csv>
df_obj_nona=df_obj.dropna(axis=0)
 df_obj_MasVnrType_na=df_obj_nona['MasVnrType']
df_obj_without_na=df_obj_nona.drop('MasVnrType',axis=1 )
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv")
 print("Train dataset has {} samples and {} attributes".format(*train.shape))
print("Test dataset has {} samples and {} attributes".format(*test.shape))<define_variables>
onehot=OneHotEncoder()
df_obj_without_na=onehot.fit_transform(df_obj_without_na )
n=len(train)
 surv_0=len(train[train['Survived']==0])
 surv_1=len(train[train['Survived']==1])
 print("% of passanger survived in train dataset: ",surv_1*100/n)
 print("% of passanger not survived in train dataset: ",surv_0*100/n )<define_variables>
knn_or=OneVsRestClassifier(KNeighborsClassifier(weights='distance',n_neighbors=8))
knn_or.fit(X=df_obj_without_na,y=df_obj_MasVnrType_na)
 Mas_pred=knn_or.predict(onehot.fit_transform(df_obj_without))
cat=['Pclass','Sex','Embarked']
num=['Age','SibSp','Parch','Fare']<compute_test_metric>
df_obj.loc[df_obj.MasVnrType.isnull() ==True,'MasVnrType']=Mas_pred[df_obj.MasVnrType.isnull() ]
csq=chi2_contingency(pd.crosstab(train['Survived'], train['Sex']))
print("P-value: ",csq[1] )<compute_test_metric>
df_obj['y']=train_y
csq2=chi2_contingency(pd.crosstab(train['Survived'], train['Embarked']))
print("P-value: ",csq2[1] )<compute_test_metric>
one_hot_=[]
for col in list(df_obj.columns)[:-1]:
tmp=df_obj[[col,'y']]
tmp=tmp.groupby(col ).mean().reset_index()
if(max(tmp.y)-min(tmp.y)) <=(2*np.std(train_y)) :
one_hot_.append(col)
 len(one_hot_ )
csq3=chi2_contingency(pd.crosstab(train['Survived'], train['Pclass']))
print("P-value: ",csq3[1] )<count_missing_values>
df_obj_one=df_obj[one_hot_]
df_obj_ordinal=df_obj.drop(one_hot_,axis=1 )
print(train.isnull().sum() )<count_missing_values>
df_obj_one=onehot.fit_transform(df_obj_one )
print(test.isnull().sum() )<feature_engineering>
for col in list(df_obj_ordinal.columns)[:-1]:
tmp=df_obj_ordinal[[col,'y']].groupby(col ).mean().reset_index()
tmp['y']=tmp['y']/sum(tmp['y'])
 change={}
for i,j in zip(tmp[col],tmp['y']):
change[i]=j
df_obj_ordinal[col]=df_obj_ordinal[col].replace(change )
med=np.nanmedian(train['Age'])
 train['Age']=train['Age'].fillna(med)
 test['Age']=test['Age'].fillna(med )<count_values>
df_obj_ordinal=df_obj_ordinal.drop('y',axis=1 )
train['Cabin'].value_counts()<data_type_conversions>
df_obj=np.concatenate(( df_obj_ordinal,df_obj_one.toarray()),axis=1 )
train['Cabin']=train['Cabin'].fillna(0)
 test['Cabin']=test['Cabin'].fillna(0 )<count_values>
df_all=np.concatenate(( df_num,df_obj),axis=1 )
train['Embarked'].value_counts()<data_type_conversions>
df_train=df_all[:1460,:]
df_test=df_all[1460:,:]
train['Cabin']=train['Cabin'].fillna("S" )<feature_engineering>
rid=Ridge(alpha=10)
 rid.fit(df_train,train_y)
 y_pred=rid.predict(df_train )
med=np.nanmedian(train['Fare'])
 test['Fare']=test['Fare'].fillna(med )<feature_engineering>
data=pd.DataFrame({'y_pred':y_pred,'y_actual':train_y} )
train['hasCabin']=train['Cabin'].apply(lambda x: 0 if x==0 else 1)
 test['hasCabin']=test['Cabin'].apply(lambda x: 0 if x==0 else 1 )<feature_engineering>
regression=ols('y_actual~y_pred',data=data ).fit()
test=regression.outlier_test()
train['FamilyMem']=train.apply(lambda x: x['SibSp']+x['Parch'], axis=1)
 test['FamilyMem']=test.apply(lambda x: x['SibSp']+x['Parch'], axis=1 )<feature_engineering>
test.sort_values('unadj_p')[:10]
train['title']=train['Name'].apply(get_title)
 test['title']=test['Name'].apply(get_title )<count_values>
remain_index=test.loc[test['bonf(p)']>=0.9].index
title_lev1=list(train['title'].value_counts().reset_index() ['index'])
 title_lev2=list(test['title'].value_counts().reset_index() ['index'] )<merge>
df_train=df_train[list(remain_index)]
title_lev=list(set().union(title_lev1, title_lev2))
print(title_lev )<feature_engineering>
train_y=train_y[list(remain_index)]
train['title']=pd.Categorical(train['title'], categories=title_lev)
 test['title']=pd.Categorical(test['title'], categories=title_lev )<define_variables>
pls=PLSRegression()
best_clf = GridSearchCV(pls,scoring='neg_root_mean_squared_error',cv=10,n_jobs=1,
param_grid={'n_components':[2,4,8,16]})
 best_clf.fit(df_train,train_y )
cols=['Pclass','Sex','Embarked','hasCabin','title']
fcol=['Pclass','Sex','Embarked','hasCabin','title','Age','FamilyMem','Fare']<data_type_conversions>
best_clf.best_score_
for c in cols:
train[c]=train[c].astype('category')
 test[c]=test[c].astype('category' )<create_dataframe>
best_clf.best_params_
train_df=train[fcol]
test_df=test[fcol]<categorify>
rid=Ridge()
best_clf = GridSearchCV(rid,scoring='neg_root_mean_squared_error',cv=10,n_jobs=1,
param_grid={'alpha': [0.01,0.1,1,10]})
 best_clf.fit(df_train,train_y )
train_df=pd.get_dummies(train_df, columns=cols, drop_first=True)
 test_df=pd.get_dummies(test_df, columns=cols, drop_first=True )<prepare_x_and_y>
best_clf.best_score_
y=train['Survived']<split>
best_clf.best_params_
x_train, x_test, y_train, y_test = train_test_split(train_df, y, test_size=0.3, random_state=42 )<choose_model_class>
svr=SVR(kernel='poly',degree=2)
 best_m=GridSearchCV(svr,param_grid={'C':[0.0001,0.001],'epsilon':[0.1,0.2,0.3],'gamma':[1,2]},
cv=10,n_jobs=3,scoring='neg_root_mean_squared_error')
 best_m.fit(df_train,train_y )
rfc=RandomForestClassifier(random_state=42 )<define_search_space>
best_m.best_params_
param_grid = {
'n_estimators': [200, 500],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth' : [4,5,6,7,8],
'criterion' :['gini', 'entropy']
}<train_on_grid>
best_m.best_score_
CV_rfc = GridSearchCV(estimator=rfc, param_grid=param_grid, cv= 5)
 CV_rfc.fit(x_train, y_train )<find_best_params>
reg=StackingRegressor(estimators=[('svm',SVR(kernel='poly',degree=2,C=0.0001,epsilon=0.1,gamma=1)) , 
('ridge',Ridge(alpha=10)) ],
final_estimator=PLSRegression(n_components=2))
cv=cross_validate(reg,X=df_train,y=train_y,cv=10,scoring='neg_root_mean_squared_error',n_jobs=1 )
CV_rfc.best_params_<choose_model_class>
reg=StackingRegressor(estimators=[('svm',SVR(kernel='poly',degree=2,C=0.0001,epsilon=0.1,gamma=1)) , 
('ridge',Ridge(alpha=10)) ],
final_estimator=PLSRegression(n_components=2))
reg.fit(df_train,train_y)
 y_pred=reg.predict(df_test )
rfc1=RandomForestClassifier(random_state=42, max_features='auto', n_estimators= 200, max_depth=8, criterion='gini' )<train_model>
y_pred=np.expm1(y_pred)
 y_pred=pd.DataFrame(y_pred)
 y_pred=y_pred.reset_index()
y_pred.columns=['Id','SalePrice']
y_pred['Id']=range(1461,2920 )
rfc1.fit(x_train, y_train )<predict_on_test>
submission=y_pred.copy()
q1 = submission['SalePrice'].quantile(0.005)
 q2 = submission['SalePrice'].quantile(0.995)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1 )
<compute_test_metric><EOS> 
submission.to_csv('./y_pred.csv',index=False )