Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<feature_engineering>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib
import seaborn as sns
from scipy.stats import norm
from scipy import stats
eng['hypothesis_misspelled'] = eng.hypothesis.apply(lambda sentence: tuple(spell.unknown(re.split('[\?!\:;,.\-\% \b\s() "/$0-9]',re.sub('['\[\]]', '', sentence)))))
 misspelled_hyp_df = eng.loc[eng.hypothesis_misspelled !=() ]
list(misspelled_hyp_df.hypothesis_misspelled )<filter>
df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 corrmat=df_train.corr()
f, ax = plt.subplots(figsize=(10,10))
sns.heatmap(corrmat, vmax=1, square=True )
eng.loc[eng.hypothesis.str.contains('Ile de R', case = False)]<feature_engineering>
total=df_train.isnull().sum().sort_values(ascending=False)
 percent=(df_train.isnull().sum() /df_train.isnull().count() ).sort_values(ascending=False)
 missing_data=pd.concat([total,percent], axis=1, keys=['Total','Percent'])
 missing_data.loc[missing_data['Total']>0].head(len(df_train.columns))
eng.loc[eng.hypothesis == 'Ile de R is no longer part of the attraction.', 'hypothesis'] = 'Ile de Re is no longer part of the attraction.'
eng.loc[eng.hypothesis == 'Ile de R.', 'hypothesis'] = 'Ile de Re.'<feature_engineering>
df_test= pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 total=df_test.isnull().sum().sort_values(ascending=False)
 percent=(df_test.isnull().sum() /df_test.isnull().count() ).sort_values(ascending=False)
 missing_data2=pd.concat([total,percent], axis=1, keys=['Total','Percent'])
 missing_data2.loc[missing_data2['Total']>0].head(len(df_test.columns))
square_brackets = eng.loc[eng.premise.str.contains('[\[\]]', case = False)]
for i in square_brackets.index:
eng.loc[eng.index == i, 'premise'] = re.sub('[\[\]]', '', str(eng.loc[eng.index == i].premise.values[0]))
ampersands = eng.loc[eng.premise.str.contains('\&amp', case = False)]
for i in ampersands.index:
eng.loc[eng.index == i, 'premise'] = re.sub('\&amp', ' and ', str(eng.loc[eng.index == i].premise.values[0]))<feature_engineering>
print('Number of houses that were apparently remodelled in 1950:')
 l=list(df_train['YearBuilt'].loc[df_train['YearRemodAdd']==1950])
 print(len(l))
print('Number of houses that were apparently build in 1900:')
 l=list(df_train['YearRemodAdd'].loc[df_train['YearBuilt']==1900])
 print(len(l))
print('Number of houses that were apparently built in 1900 and remodelled in 1950:')
 print(len([year for year in l if year==1950]))
print('Fraction of houses that were remodelled:')
 print(len(list(df_train['YearRemodAdd'].loc[df_train['YearBuilt']!=df_train['YearRemodAdd']])) /len(df_train['YearRemodAdd']))
eng.drop(columns=['premise_misspelled', 'hypothesis_misspelled'])
 for i in eng.index:
train.loc[train.index == i] = eng.loc[eng.index == i]<filter>
df_train['FloorsSF']=pd.Series(len(df_train['YearBuilt']), index=df_train.index)
 df_train['FloorsSF']=df_train['1stFlrSF']+df_train['TotalBsmtSF']+df_train['2ndFlrSF']+df_train['GrLivArea']
medianG=df_train.loc[df_train['GarageCars']==1,'GarageArea'].median()
df_train['Garage']=pd.Series(len(df_train['YearBuilt']), index=df_train.index)
 df_train['Garage']=df_train['GarageCars']+1*df_train['GarageArea']/medianG
df_test['FloorsSF']=pd.Series(len(df_test['YearBuilt']), index=df_test.index)
 df_test['FloorsSF']=df_test['1stFlrSF']+df_test['TotalBsmtSF']+df_test['2ndFlrSF']+df_test['GrLivArea']
medianG=df_test.loc[df_test['GarageCars']==1,'GarageArea'].median()
df_test['Garage']=pd.Series(len(df_train['YearBuilt']), index=df_test.index)
 df_test['Garage']=df_test['GarageCars']+1*df_test['GarageArea']/medianG
corrmat=df_train.corr()
k = 11
cols = corrmat.nlargest(k, 'SalePrice')['SalePrice'].index
cm = np.corrcoef(df_train[cols].values.T)
 sns.set(font_scale=1)
 f1, ax1 = plt.subplots(figsize=(9,9))
sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)
 plt.show()
eng_test = test.loc[test.language == 'English']<feature_engineering>
df_train = df_train.drop(( missing_data[missing_data['Total'] > 0] ).index,1)
 df_test = df_test.drop(( missing_data[missing_data['Total'] > 0] ).index,1)
 index_list_noncat_exclude=['GrLivArea','1stFlrSF','GarageCars','GarageArea','TotalBsmtSF','2ndFlrSF','HouseStyle', 'Utilities', 'Exterior2nd', 'Condition2', 'Heating', 'Exterior1st', 'RoofMatl']
df_train = df_train.drop(index_list_noncat_exclude,1)
 df_test = df_test.drop(index_list_noncat_exclude,1)
 print('Print zero, if no data is missing in the new training set: ')
 df_train.isnull().sum().max()
err1 = eng_test.loc[eng_test.premise.str.contains('\?\?', case = False)]
correction = err1.premise.apply(lambda sentence: re.sub('\?\?', 'e', sentence))
correct_alacahoyuk = correction.apply(lambda sentence: re.sub('Alacaheyek', 'Alacahoyuk', sentence))
correct_madrileno = correct_alacahoyuk.apply(lambda sentence: re.sub('Madrileeo', 'Madrileno', sentence))
correct_alcudia = correct_madrileno.apply(lambda sentence: re.sub('Alcedia', 'Alcudia', sentence))
correct_ataturk = correct_alcudia.apply(lambda sentence: re.sub('Ataterk', 'Ataturk', sentence))
for i in correct_ataturk.index:
eng_test.loc[eng_test.index == i, 'premise'] = correct_ataturk.loc[correct_ataturk.index == i]<feature_engineering>
df_train['Centenarian']=pd.Series(len(df_train['YearBuilt']), index=df_train.index)
 df_train['Centenarian']=0
df_train.loc[df_train['YearBuilt']<1920,'Centenarian']=1
df_test['Centenarian']=pd.Series(len(df_test['YearBuilt']), index=df_test.index)
 df_test['Centenarian']=0
df_test.loc[df_test['YearBuilt']<1920,'Centenarian']=1
df_train['Remodelled']=pd.Series(len(df_train['YearBuilt']), index=df_train.index)
 df_train['Remodelled']=0
df_train.loc[df_train['YearBuilt']!=df_train['YearRemodAdd'],'Remodelled']=1
df_test['Remodelled']=pd.Series(len(df_test['YearBuilt']), index=df_test.index)
 df_test['Remodelled']=0
df_test.loc[df_test['YearBuilt']!=df_test['YearRemodAdd'],'Remodelled']=1
f1, ax1 = plt.subplots(figsize=(10, 10))
fig = sns.boxplot(x='Centenarian', y="SalePrice", data=df_train)
 f3, ax3 = plt.subplots(figsize=(10, 10))
fig = sns.boxplot(x='Remodelled', y="SalePrice", data=df_train )
err2 = eng_test.loc[eng_test.premise.str.contains('\?\xad', case = False)]
correction = err2.premise.apply(lambda sentence: re.sub('\?\xad', '', sentence))
for i in correction.index:
eng_test.loc[eng_test.index == i, 'premise'] = correction.loc[correction.index == i]<feature_engineering>
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, train, y, scoring="neg_mean_squared_error", cv = 10))
return(rmse)
 df_test = df_test.fillna(df_test.mean())
 df_train = pd.get_dummies(df_train)
 test = pd.get_dummies(df_test)
 cols=list(df_train.columns)
 cols.remove('SalePrice')
 train=df_train[cols]
y=df_train.SalePrice
model_lasso = LassoCV(alphas = [0.5, 0.005, 0.0005] ).fit(train, y)
 print('RMSE on the train set:')
 rmse_cv(model_lasso ).mean()
square_brackets = eng_test.loc[eng_test.premise.str.contains('[\[\]]', case = False)]
for i in square_brackets.index:
eng_test.loc[eng_test.index == i, 'premise'] = re.sub('[\[\]]', '', str(eng_test.loc[eng_test.index == i].premise.values[0]))<feature_engineering>
dtrain = xgb.DMatrix(train, label = y)
 dtest = xgb.DMatrix(test)
 params = {"max_depth":2, "eta":0.1}
model = xgb.cv(params, dtrain, num_boost_round=500, early_stopping_rounds=200)
 model_xgb = xgb.XGBRegressor(n_estimators=360, max_depth=2, learning_rate=0.1)
 model_xgb.fit(train, y)
 xgb_preds = np.expm1(model_xgb.predict(test))
lasso_preds = np.expm1(model_lasso.predict(test))
predictions = pd.DataFrame({"xgb":xgb_preds, "lasso":lasso_preds})
 predictions.plot(x = "xgb", y = "lasso", kind = "scatter" )
ampersands = eng_test.loc[eng_test.premise.str.contains('\&amp', case = False)]
for i in ampersands.index:
eng_test.loc[eng_test.index == i, 'premise'] = re.sub('\&amp', ' and ', str(eng_test.loc[eng_test.index == i].premise.values[0]))<feature_engineering>
preds = 0.5*lasso_preds + 0.5*xgb_preds
solution = pd.DataFrame({"id":df_test.Id, "SalePrice":preds} )
<save_to_csv><EOS> 
solution.to_csv('result-with-best.csv', index=False )