Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_from_csv>
import pandas as pd;
import numpy as np;
import seaborn as sns;
import matplotlib.pyplot as plt;
test_df = pd.read_csv('.. /input/titanic/test.csv' )<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv');
combine = [train_df, test_df]<drop_column>
ids = test['Id']
train_df = train_df.drop(['Name', 'PassengerId'], axis=1)
 test_df = test_df.drop(['Name'], axis=1)
 combine = [train_df, test_df]
train_df.shape, test_df.shape<data_type_conversions>
if 'SalePrice' in train:
y = train['SalePrice']
if 'Id' in test:
ids = test.pop('Id')
 train.pop('Id' )
for dataset in combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 train_df.head()<feature_engineering>
dataset = pd.concat([train, test],axis=0,ignore_index=True)
 dataset.reset_index(drop=True)
 print(dataset.shape)
for dataset in combine:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1<drop_column>
dataset = dataset.reindex(index=range(dataset.shape[0]))
train_df = train_df.drop(['Parch', 'SibSp'], axis=1)
 test_df = test_df.drop(['Parch', 'SibSp'], axis=1)
 combine = [train_df, test_df]
train_df.head()<correct_missing_values>
null_alley = dataset.loc[dataset['Alley'].isnull() ,'Alley']
null_num = len(null_alley)
 L = [ 'Grvl', 'Pave']
np.random.seed(125)
 random_list = np.random.random_integers(1,null_num,null_num)
 random_val = [ L[ x % len(L)] for x in random_list]
dataset.loc[dataset['Alley'].isnull() ,'Alley'] = random_val
freq_port = train_df.Embarked.dropna().mode() [0]
freq_port<data_type_conversions>
Bsmt_fl = ['BsmtCond','BsmtExposure','BsmtFinSF1','BsmtFinSF2','BsmtFinType1','BsmtFinType2','BsmtFullBath','BsmtHalfBath','BsmtQual','BsmtUnfSF']
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port )<data_type_conversions>
all_null_basement = dataset['BsmtCond'].isnull() &\
dataset['BsmtExposure'].isnull() &\
dataset['BsmtFinType1'].isnull() &\
dataset['BsmtFinType2'].isnull() &\
dataset['BsmtQual'].isnull() &\ 
(dataset['BsmtFinSF1'] ==0.0)&\ 
(dataset['BsmtFinSF2'] ==0.0)&\ 
(dataset['BsmtUnfSF'] == 0)&\ 
(dataset['BsmtFullBath'] == 0)&\ 
(dataset['BsmtHalfBath'] == 0)
 print('all null and o sum is' ,sum(all_null_basement.where(all_null_basement==True,0)) )
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_df.head()<data_type_conversions>
all_notnull_basement =(all_null_basement == False )
test_df['Fare'].fillna(test_df['Fare'].dropna().median() , inplace=True)
 test_df.head()<define_search_space>
Bsmt_catigorail_fl = ['BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','BsmtQual']
guess_ages = np.zeros(( 2,3))
guess_ages
for dataset in combine:
for i in range(0, 2):
for j in range(0, 3):
guess_df = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
guess_ages[i,j] = int(age_guess/0.5 + 0.5)* 0.5
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]
dataset['Age'] = dataset['Age'].astype(int)
<prepare_x_and_y>
dataset.loc[dataset['BsmtCond'].isnull() , Bsmt_fl]
X_train = train_df.drop("Survived", axis=1)
 Y_train = train_df["Survived"]
X_test = test_df.drop("PassengerId", axis=1 ).copy()
X_train.shape, Y_train.shape, X_test.shape<import_modules>
dataset.loc[2040,['BsmtCond']]='Ex'
dataset.loc[[2120,2185],['BsmtCond']]='TA'
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier<compute_train_metric>
dataset[Bsmt_fl].isnull().sum()
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 Y_pred = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2)
 acc_random_forest<normalization>
dataset.loc[dataset['BsmtExposure'].isnull() ,Bsmt_fl]
scaler = StandardScaler()
scaler.fit(X_train)
 X_train = scaler.transform(X_train)
 X_test = scaler.transform(X_test )<import_modules>
dataset[Bsmt_catigorail_fl].isnull().sum()
from sklearn.neural_network import MLPClassifier<train_model>
dataset.loc[dataset['BsmtFinType2'].isnull() ,Bsmt_fl]
clf = MLPClassifier(solver='lbfgs', alpha=1e-5,hidden_layer_sizes=(5, 2), random_state=1)
 clf.fit(X_train, Y_train)
 MLPClassifier(alpha=1e-05, hidden_layer_sizes=(5, 2), random_state=1, solver='lbfgs')
 Y_pred = clf.predict(X_test)
 Y_pred<create_dataframe>
dataset.loc[[332],['BsmtFinType2']]='GLQ'
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_pred
} )<save_to_csv>
dataset.loc[dataset['BsmtQual'].isnull() ,Bsmt_fl]
submission.to_csv('my_submission1.csv', index=False )<set_options>
dataset.loc[[2217],['BsmtQual']]='Fa'
dataset.loc[[2218],['BsmtQual']]='TA'
pd.options.mode.chained_assignment = None<load_from_csv>
dataset[Bsmt_catigorail_fl].isnull().sum()
train=pd.read_csv('/kaggle/input/titanic/train.csv')
 train.head()<load_from_csv>
dataset[Bsmt_fl].isnull().sum()
test=pd.read_csv('/kaggle/input/titanic/test.csv')
 pid=test.PassengerId<count_missing_values>
dataset.loc[dataset['BsmtFinSF1'].isnull() ,Bsmt_fl]
print('Train Set')
 print(train[train.isnull() ])
 print('
Test Set')
 print(test.isnull().sum() )<data_type_conversions>
dataset.loc[dataset['BsmtFinSF1'].isnull() ,'BsmtFinSF1']=0
dataset.loc[dataset['BsmtFinSF2'].isnull() ,'BsmtFinSF2']=0
dataset.loc[dataset['BsmtFullBath'].isnull() ,'BsmtFullBath']=0
dataset.loc[dataset['BsmtHalfBath'].isnull() ,'BsmtHalfBath']=0
dataset.loc[dataset['BsmtUnfSF'].isnull() ,'BsmtUnfSF']=0
train['Age']=train['Age'].fillna(train['Age'].median())
 train['Embarked']=train['Embarked'].fillna(train['Embarked'].mode() [0])
 test['Age']=test['Age'].fillna(test['Age'].median())
 test['Fare']=test['Fare'].fillna(test['Fare'].mean() )<count_missing_values>
dataset[Bsmt_fl].isnull().sum()
print('Train Set')
 print(train.isnull().sum())
 print('
Test Set')
 print(test.isnull().sum() )<feature_engineering>
dataset.loc[dataset['Electrical'].isnull() ,'Electrical']='FuseA'
for data in [train,test]:
data['FamilySize']=data['SibSp']+data['Parch']+1
data['IsAlone']=1
data['IsAlone'].loc[data['FamilySize']>1]=0
data['Title'] = data['Name'].str.split(", ", expand=True)[1].str.split(".", expand=True)[0]<count_values>
ex=dataset.loc[dataset['Exterior1st'].isnull() ]
train['Title'].value_counts()<feature_engineering>
dataset.loc[dataset['Exterior1st'].isnull() ,'Exterior1st']='VinylSd'
train['Title']=train['Title'].replace('Ms','Miss')
 train['Title']=train['Title'].replace('Mlle','Miss')
 train['Title']=train['Title'].replace('the Countess','Mrs')
 train['Title']=train['Title'].replace('Mme','Mrs')
 test['Title']=test['Title'].replace('Dona','Mrs')
 test['Title']=test['Title'].replace('Ms','Miss')
 names=(train['Title'].value_counts() < 10)
 train['Title']=train['Title'].apply(lambda x: 'Misc' if names.loc[x] == True else x)
 names=(test['Title'].value_counts() < 10)
 test['Title']=test['Title'].apply(lambda x: 'Misc' if names.loc[x] == True else x)
 print('Train Set
',train['Title'].value_counts())
 print('
Test Set
',test['Title'].value_counts() )<drop_column>
dataset.loc[dataset['Exterior2nd'].isnull() ,'Exterior2nd']='VinylSd'
columns=['PassengerId','Cabin','Ticket','Name']
train=train.drop(columns,axis=1)
 test=test.drop(columns,axis=1 )<categorify>
dataset['Fence_num']=dataset['Fence'].cat.codes
dataset['Fence_num'] = dataset['Fence_num'].astype(int )
label=LabelEncoder()
for data in [train,test]:
data['Sex']=label.fit_transform(data['Sex'])
 data['Embarked']=label.fit_transform(data['Embarked'])
 data['Title']=label.fit_transform(data['Title'])
 data['Age']=data['Age'].astype('int64')
 train.head()<groupby>
fence_notnull_set = dataset.loc[dataset['Fence'].isnull() ==False]
lens = len(fence_notnull_set['Fence'])
 print(lens)
 probs=(fence_notnull_set['Fence'].value_counts() / lens ).tolist()
print(probs )
target=['Survived']
selected=['Sex','Pclass','Embarked','Title','SibSp','Parch','FamilySize','IsAlone']
for x in selected:
print('Survival Percentage By',x)
 print(train[[x, target[0]]].groupby(x,as_index=False ).mean() ,'
')
<prepare_x_and_y>
fence_t = dataset['Fence'].cat.categories.tolist()
fence_null_nums = len(dataset.loc[dataset['Fence'].isnull() ])
 fence_null = np.random.choice(fence_t,fence_null_nums,p=probs)
 dataset.loc[dataset['Fence'].isnull() ,'Fence'] = fence_null
dataset['Fence'].unique()
target=train['Survived']
train.drop(['Survived'],axis=1, inplace=True )<train_model>
np.random.randint(10,1000,1)[0]
X_train, X_val, y_train, y_val = train_test_split(train, target, test_size=0.2, random_state=1, stratify=target)
 print('Mean Absolute Errors:')
 model = RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,
criterion='gini', max_depth=4, max_features='auto',
max_leaf_nodes=5, max_samples=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=15,
min_weight_fraction_leaf=0.0, n_estimators=350,
n_jobs=None, oob_score=True, random_state=1, verbose=0,
warm_start=False)
 model.fit(X_train, y_train)
 predict = model.predict(X_val)
 print('Random Forrest: ' + str(mean_absolute_error(predict, y_val)))
 model = XGBRegressor(learning_rate=0.01, n_estimators=3460, max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,colsample_bytree=0.7,objective='reg:squarederror',
nthread=-1,scale_pos_weight=1, seed=27, reg_alpha=0.00006)
 model.fit(X_train, y_train)
 predict = model.predict(X_val)
 print('XGBoost: ' + str(mean_absolute_error(predict, y_val)))
 model = LassoCV(max_iter=1e7, random_state=14, cv=10)
 model.fit(X_train, y_train)
 predict = model.predict(X_val)
 print('Lasso: ' + str(mean_absolute_error(predict, y_val)))
 model = GradientBoostingRegressor(n_estimators=300, learning_rate=0.05, max_depth=4, random_state=5)
 model.fit(X_train, y_train)
 predict = model.predict(X_val)
 print('GradientBoosting: ' + str(mean_absolute_error(predict, y_val)) )<choose_model_class>
dataset.loc[dataset['LandContour']=='HLS','Fence' ] ='MnPrv'
dataset.loc[dataset['LotConfig']=='RH','Fence' ] ='MnPrv'
dataset.loc[dataset['LandSlope']=='Sev','Fence' ] ='GdWo'
dataset.loc[(dataset['Condition1']=='RRNn')|(dataset['Condition1']=='PosA')|(dataset['Condition1']=='RRne'),'Fence' ] ='GdPrv'
dataset.loc[(dataset['Condition2']=='PosN'),'Fence' ] ='GdPrv'
dataset.loc[(dataset['MSZoning']=='RH'),'Fence' ] ='GdPrv'
tmp = dataset.loc[dataset['MSZoning']==r'C(all)','Fence']
d = ['GdWo','MnPrv']
lens = len(tmp)
 dataset.loc[dataset['MSZoning']==r'C(all)','Fence'] = [ d[ x % len(d)] for x in np.random.randint(10,1000,lens)]
tmp = dataset.loc[dataset['MSZoning']==r'FV','Fence']
d = ['GdWo','GdPrv']
lens = len(tmp)
 dataset.loc[dataset['MSZoning']==r'FV','Fence'] = [ d[ x % len(d)] for x in np.random.randint(10,1000,lens)]
model=RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,
criterion='gini', max_depth=4, max_features='auto',
max_leaf_nodes=5, max_samples=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=15,
min_weight_fraction_leaf=0.0, n_estimators=350,
n_jobs=None, oob_score=True, random_state=1, verbose=0,
warm_start=False)
 model.fit(X_train, y_train)
 predict = model.predict(X_val)
 print('Random Forest MAE: ' + str(mean_absolute_error(predict, y_val)))
 print("Out of Bag Score: %.4f" % model.oob_score_)
 y_pred_train = model.predict(X_train)
 y_pred_test = model.predict(X_val)
 print("Training accuracy: ", accuracy_score(y_train, y_pred_train))
print("Testing accuracy: ", accuracy_score(y_val, y_pred_test))
print("
Confusion Matrix
")
 print('[[True Positive False Positive]
[False Negative True Negative]]
')
 print(confusion_matrix(y_val, y_pred_test))
fpr, tpr, _ = roc_curve(y_val, y_pred_test)
 roc_auc = auc(fpr, tpr)
 print("
ROC AUC on evaluation set",roc_auc)
 plt.figure()
plt.plot(fpr, tpr, label='ROC curve(area = %0.2f)' % roc_auc)
 plt.plot([0, 1], [0, 1], 'k--')
 plt.xlim([0.0, 1.0])
 plt.ylim([0.0, 1.05])
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.title('ROC Curve')
 plt.legend(loc="lower right")
 plt.show()<predict_on_test>
dataset.drop(labels=['Fence_num'],axis=1,inplace=True )
prediction=model.predict(test)
 prediction<save_to_csv>
tmp = dataset.loc[dataset['FireplaceQu'].isnull() ]
d = ['TA','Gd']
lens = len(tmp)
 dataset.loc[dataset['FireplaceQu'].isnull() ,'FireplaceQu'] = [ d[ x % len(d)] for x in np.random.randint(10,1000,lens)]
output = pd.DataFrame({'PassengerId': pid, 'Survived': prediction})
 output.to_csv('my_submission.csv', index=False)
 print("Submission successfully saved!" )<import_modules>
dataset.loc[dataset['MSZoning'].isnull() ,'MSZoning'] = 'RL';
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sklearn.model_selection as ms
import seaborn as sns
import sklearn.linear_model as lm
import sklearn.metrics as m
import sklearn.dummy as d
import sklearn.preprocessing as p
import sklearn.ensemble as e
import sklearn.pipeline as pipeline
import sklearn.svm as svm
import sklearn.impute as impute
from sklearn.compose import make_column_transformer
import scipy.stats as stats
import lightgbm as lgb
import xgboost as xgb<set_options>
dataset.loc[dataset['MasVnrArea'].isnull() ,'MasVnrArea'] = dataset['MasVnrArea'].median()
plt.figure(figsize=(3,3))<set_options>
dataset.loc[dataset['MasVnrType'].isnull() ]
sns.set(rc={'figure.figsize':(20,15)} )<load_from_csv>
dataset.loc[dataset['MasVnrType'].isnull() ,'MasVnrType'] = 'None'
hp = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv' )<count_values>
MiscFeature_notnull_set = dataset.loc[dataset['MiscFeature'].isnull() ==False]
def print_types_in_df(df):
types = ['object', 'category', 'int64', 'float64']
types_to_counts = {x: len(df.select_dtypes(include=[x] ).columns.values)for x in types}
print(types_to_counts)
 print('sum: ' +str(sum(types_to_counts.values())) )<define_variables>
dataset.loc[ dataset['MiscFeature'].isnull() ,'MiscFeature']='Shed'
categorical_oh_features = ['MSSubClass', 'MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'RoofStyle',
'RoofMatl', 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'Foundation', 'Heating', 'Electrical', 'Functional', 'GarageType', 'MiscFeature', 'SaleType', 'SaleCondition']
categorical_l_features = ['OverallQual', 'OverallCond', 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'HeatingQC', 'CentralAir', 'KitchenQual', 'FireplaceQu', 'GarageFinish', 'GarageQual', 'GarageCond', 'PavedDrive', 'PoolQC', 'Fence']
numerical_features = ['LotFrontage', 'LotArea', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces', 'GarageCars', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal']
date_features = ['YearBuilt', 'YearRemodAdd', 'GarageYrBlt', 'MoSold', 'YrSold']
unused_features = ['Id']
target_feature = ['SalePrice']<define_variables>
PoolQC_notnull_set = dataset.loc[dataset['PoolQC'].isnull() == False]
len(PoolQC_notnull_set )
len(categorical_oh_features)+ len(categorical_l_features)+ len(numerical_features)+ len(date_features)+ len(unused_features)+ len(target_feature )<data_type_conversions>
if 'PoolQC' in dataset.columns:
dataset.drop(labels='PoolQC',axis=1,inplace=True);
hp[categorical_oh_features] = hp[categorical_oh_features].astype('category' )<data_type_conversions>
dataset.loc[dataset['TotalBsmtSF'].isnull() ,'TotalBsmtSF'] = dataset['TotalBsmtSF'].median()
hp[categorical_l_features] = hp[categorical_l_features].astype('category' )<data_type_conversions>
dataset.loc[dataset['Utilities'].isnull() ,'Utilities'] = 'AllPub'
hp[numerical_features] = hp[numerical_features].apply(pd.to_numeric, errors='coerce' )<drop_column>
dataset.loc[dataset['SaleType'].isnull() ,'SaleType'] = 'Other'
hp.drop(columns=unused_features, inplace=True )<count_unique_values>
dataset.loc[dataset['LotFrontage'].isnull() ,'LotFrontage'] = dataset['LotFrontage'].median()
discrete_cols=[]
for col in hp.columns:
if hp[col].dtype=='int64' and len(hp[col].unique())<=15:
discrete_cols.append(col)
 discrete_cols<define_variables>
dataset.loc[dataset['GarageArea'].isnull() ,'GarageArea'] = dataset['GarageArea'].median()
continuous_cols=[]
for col in hp.columns:
if(hp[col].dtype=='int' or hp[col].dtype=='float')and col not in ['Id', 'YearBuilt','YearRemodAdd','GarageYrBlt'] and col not in discrete_cols:
continuous_cols.append(col)
 continuous_cols<sort_values>
dataset.loc[dataset['GarageCars'].isnull() ,'GarageCars'] = dataset['GarageCars'].median()
t = abs(corr.SalePrice ).sort_values(ascending=False)
 print(t)
 t.index<define_variables>
dataset.loc[dataset['Functional'].isnull() ,'Functional']='Typ'
cat1 = ['MSSubClass']<create_dataframe>
dataset.loc[dataset['KitchenQual'].isnull() ,'KitchenQual']='TA'
pt = hp.pivot_table(index='MSSubClass', values='SalePrice', aggfunc=np.median )<sort_values>
Garage_fl = ['GarageCond','GarageQual','GarageArea','GarageCars','GarageFinish','GarageType']
Garage_cat_fl = ['GarageCond','GarageQual','GarageFinish','GarageType']
pt.sort_values(by='SalePrice',ascending=False )<define_variables>
null_cond =(dataset['GarageCond'].isnull())|\ 
(dataset['GarageQual'].isnull())|\ 
(dataset['GarageFinish'].isnull())|\ 
(dataset['GarageType'].isnull())
 Garage_null_set=dataset.loc[null_cond]
cat2=['MSZoning','Street','LotShape','LandContour','LotConfig','LandSlope']<define_variables>
not_null_cond =(Garage_null_set['GarageCond'].isnull() == False)|\ 
(Garage_null_set['GarageQual'].isnull() == False)|\ 
(Garage_null_set['GarageArea'] != 0)|\ 
(Garage_null_set['GarageCars'] != 0)|\ 
(Garage_null_set['GarageFinish'].isnull() == False)|\ 
(Garage_null_set['GarageType'].isnull() == False)
 Garage_null_set.loc[ not_null_cond,Garage_fl ]
year_features = ['YearBuilt','YearRemodAdd','YrSold','GarageYrBlt']<create_dataframe>
dataset.loc[dataset['GarageYrBlt'].isnull() ,'GarageYrBlt'] = dataset.loc[dataset['GarageYrBlt'].isnull() ,'YearBuilt']
t = pd.DataFrame()<feature_engineering>
if 'SalePrice' in dataset:
dataset.drop(labels='SalePrice',axis=1,inplace=True )
t['SalePrice'] = hp.SalePrice
t['YrSold'] = hp.YrSold
t['YearBuilt'] = hp.YrSold - hp.YearBuilt
t['YearRemodAdd'] = hp.YrSold - hp.YearRemodAdd
t['GarageYrBlt'] = hp.YrSold - hp.GarageYrBlt<create_dataframe>
sum(dataset.isnull().sum() )
nulls = hp.isna().sum().sort_values(ascending=False)
 nulls_df = pd.DataFrame({'feature': nulls.index, 'value': nulls.values})
 nulls_df<concatenate>
year_fl = ['YearBuilt','YearRemodAdd','YrSold','GarageYrBlt']
nulls_with_type = pd.concat([hp[nulls_df.feature.values].dtypes, nulls], axis=1, keys=['type', 'count_of_nulls'])
 nulls_with_type[nulls_with_type.count_of_nulls > 0]<filter>
dataset.loc[(dataset['YearBuilt'] >dataset['YearRemodAdd'])|(dataset['YearRemodAdd'] >dataset['YrSold']),year_fl]
numericals_nulls = nulls_with_type[nulls_with_type.index.isin(numerical_features)]
numericals_nulls<count_missing_values>
dataset['YearBuiltTillNow'] = 2018-dataset['YearBuilt']
dataset['YearRemodTillNow'] = 2018-dataset['YearRemodAdd']
dataset['YearSoldTillNow'] = 2018-dataset['YrSold']
dataset['YearGaBuiltNow'] = 2018-dataset['GarageYrBlt']
dataset['RemodAfterBuilt'] = dataset['YearBuiltTillNow'] - dataset['YearRemodTillNow']
dataset['SoldAfterBuilt'] = dataset['YearBuiltTillNow'] - dataset['YearSoldTillNow']
dataset.loc[dataset['YearGaBuiltNow'] < 0,'YearGaBuiltNow'] = dataset.loc[dataset['YearGaBuiltNow'] < 0,'YearBuilt']
for x in year_fl:
if x in dataset:
dataset.drop(labels=x,axis=1,inplace= True )
hp[numerical_features].isnull().sum()<feature_engineering>
cat_dataset = dataset.select_dtypes(include=['object','bool'] )
numerical_imputer = impute.SimpleImputer(missing_values=np.nan, strategy='median')
 hp[numerical_features] = numerical_imputer.fit_transform(hp[numerical_features] )<count_missing_values>
for col in cat_dataset:
cat_dataset[col] = cat_dataset[col].astype('category')
hp[numerical_features].isnull().sum().sum()<filter>
num_dataset = dataset.select_dtypes(exclude=['object','category'] ).astype('float64' )
date_nulls = nulls_with_type[nulls_with_type.index.isin(date_features)]
date_nulls<filter>
pos = np.where(num_dataset < 0)
 pos[0]
date_nulls_above_zero = date_nulls[date_nulls.count_of_nulls > 0]
date_nulls_above_zero<categorify>
num_dataset.loc[ num_dataset['RemodAfterBuilt'] < 0,'RemodAfterBuilt'] = 0
num_dataset.loc[ num_dataset['SoldAfterBuilt'] < 0,'SoldAfterBuilt'] = 0
date_imputer = impute.SimpleImputer(missing_values=np.nan, strategy='median')
 hp[date_features] = date_imputer.fit_transform(hp[date_features] )<count_missing_values>
pos = np.where(num_dataset < 0)
 pos[0]
hp[date_features].isnull().sum().sum()<define_variables>
skew_feats = skew_feats[abs(skew_feats)> 1]
print(skew_feats)
 for feat in skew_feats.index:
num_dataset[feat] = np.log1p(num_dataset[feat] )
categorical_features_to_impute_none = []<define_variables>
cat_dataset = pd.get_dummies(cat_dataset, columns = cat_dataset.columns);
_features_to_drop = ['PoolQC','MiscFeature','Alley','Fence']<concatenate>
np.where(num_dataset < 0 )
unused_features.extend(_features_to_drop )<sort_values>
for x in num_dataset:
num_dataset[x] =(num_dataset[x] - num_dataset[x].mean())/(num_dataset[x].std())
categorical_oh_features = list(set(categorical_oh_features ).difference(set(_features_to_drop)) )<sort_values>
sum(np.isnan(num_dataset ).any() )
categorical_l_features = list(set(categorical_l_features ).difference(set(_features_to_drop)) )<drop_column>
np.where(np.isnan(num_dataset))
hp.drop(columns=_features_to_drop,axis=1,inplace=True )<filter>
sum(np.isnan(num_dataset ).any() )
categorical_oh_nulls = nulls_with_type[nulls_with_type.index.isin(categorical_oh_features)]
categorical_oh_nulls<filter>
dataset = pd.concat([cat_dataset,num_dataset],axis=1)
 X = dataset[:train_len]
X_test = dataset[train_len:]
sum(X.isnull().sum())
categorical_l_nulls = nulls_with_type[nulls_with_type.index.isin(categorical_l_features)]
categorical_l_nulls<define_variables>
sum(X_test.isnull().sum())
 X.shape
categorical_features_to_impute_none += ['FireplaceQu']<define_variables>
cross_val_score, \
StratifiedKFold, \
learning_curve,\
KFold,\
cross_val_predict;
sns.set(style='white', context='notebook', palette='deep');
warnings.filterwarnings(action='ignore', category=UserWarning)
categorical_features_to_impute_none += ['MasVnrType']<filter>
def train_model(estimator=None, X_train=None,y_train=None,X_cv=None,y_cv=None,scoring=None):
m = estimator
if(scoring == 'neg_mean_squared_error'):
m.fit(X_train,np.log1p(y_train))
if(scoring == 'neg_mean_squared_log_error'):
m.fit(X_train,y_train)
 pred=estimator.predict(X_cv)
 if(scoring == 'neg_mean_squared_error'):
score = np.sqrt(mean_squared_error(np.log1p(y_cv),pred))
if(scoring == 'neg_mean_squared_log_error'):
score = np.sqrt(mean_squared_log_error(y_cv,pred))
return score,m
hp[hp['BsmtQual'].isna() ][['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','BsmtFinSF1',
'BsmtFinSF2','BsmtUnfSF','TotalBsmtSF']]<count_missing_values>
def gen_submission(csvname=None,X_test=None,models=None,scoring=None):
if(scoring == 'neg_mean_squared_error'):
pred=np.expm1(models.predict(X_test))
if(scoring == 'neg_mean_squared_log_error'):
pred=models.predict(X_test)
 result = pd.concat([ids,pd.Series(pred ).astype('float64')],axis=1);
result.columns = ['Id','SalePrice']
result.to_csv(csvname+r'.csv',index=False)
 pass
hp[bsmt_cols].isna().sum()<define_variables>
X = X.astype('float64' )
categorical_features_to_impute_none += bsmt_cols<filter>
sum(np.isnan(X ).all() )
hp[hp['Electrical'].isna() ]<filter>
sum(np.isinf(X ).any() )
garage_cols = ['GarageType','GarageFinish','GarageQual','GarageCond']
hp[hp['GarageType'].isna() ][garage_cols]<count_missing_values>
hp[garage_cols].isna().sum()<define_variables>
scores_cv = []
scores_test=[]
scoring= 'neg_mean_squared_error'
models = []
LR_ridge = Ridge(alpha=20)
 if scoring == 'neg_mean_squared_error':
LR_lasso = Lasso(alpha=0.0005)
 if scoring == 'neg_mean_squared_log_error':
LR_lasso = Lasso(alpha=80)
 models.append(['ridge',LR_ridge ])
 models.append(['lasso',LR_lasso])
 times = 10;
score_mat = np.empty(( times,len(models),2))
print('test vs cv score:')
 for i in range(times):
X_train, X_vld, y_train,y_vld = train_test_split(X,y,test_size=0.2,shuffle=True)
 j=0;
for n,m in models:
s,fin = train_model(m,X,y,X,y,scoring=scoring)
 score_mat[i,j,0] = s
s,fin = train_model(m,X_train,y_train,X_vld,y_vld,scoring=scoring)
 score_mat[i,j,1] = s
j = j+1
print('For models', list(map(lambda x: x[0] ,models)))
 print('Ave test score: ' ,np.mean(score_mat,axis=0)[:,0])
 print('Ave cv score: ' ,np.mean(score_mat,axis=0)[:,1] )
categorical_features_to_impute_none += garage_cols<sort_values>
scores = -1 * cross_val_score(LR_lasso,X,np.log1p(y),scoring=scoring,cv=5)
 np.sqrt(scores ).mean()
categorical_features_to_impute_mode = list(set(categorical_oh_features + categorical_l_features ).difference(set(categorical_features_to_impute_none)))
 categorical_features_to_impute_mode<correct_missing_values>
LR = Lasso()
kf = StratifiedKFold(n_splits=10,shuffle=True)
 lasso_param_grid = {
"alpha":[0.001,0.0005,0.0007] ,
"max_iter":[1000,800,500],
"tol":[0.001,0.002,0.005,0.01,0.02,0.04,],
}
rcv_param_grid = {}
gsLR = GridSearchCV(LR,param_grid = lasso_param_grid, cv=5,n_jobs= -1, verbose = 1)
 gsLR.scoring = scoring
gsLR.fit(X,np.log1p(y))
bestLR= gsLR.best_estimator_
print(bestLR)
 print(np.sqrt(-gsLR.best_score_))
<feature_engineering><EOS> 
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, X, np.log1p(y), scoring="neg_mean_squared_error", cv = 5))
return(rmse )