Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<set_options>
print(os.listdir(".. /input"))
warnings.filterwarnings('ignore')
%matplotlib inline 
<load_from_csv>
print("imported data.. ")
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print("Train set size:", train.shape)
 print("Test set size:", test.shape )
testandtrain_df = pd.read_csv('/kaggle/input/titanic/train.csv')
 validation_df = pd.read_csv('/kaggle/input/titanic/test.csv')
<train_model>
train_ID = train['Id']
test_ID = test['Id']
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True )
print("Training dataset(rows, columns):", testandtrain_df.shape)
 print("Validation dataset(rows, columns):", validation_df.shape )<filter>
y = train.SalePrice.reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 print('Features size:', features.shape )
validation_df[validation_df.Fare.isna() ]<create_dataframe>
def percent_missing(df):
data = pd.DataFrame(df)
 df_cols = list(pd.DataFrame(data))
dict_x = {}
for i in range(0, len(df_cols)) :
dict_x.update({df_cols[i]: round(data[df_cols[i]].isnull().mean() *100,2)})
 return dict_x
missing = percent_missing(features)
 df_miss = sorted(missing.items() , key=lambda x: x[1], reverse=True)
 print('Percent of missing data')
 df_miss[0:10]
temp1_df = pd.concat([testandtrain_df.copy() , validation_df.copy() ], sort = False)
 temp1_df = temp1_df.filter(["Pclass", "Fare"])
 temp1_df = temp1_df.dropna()
print("Combined dataframe of all passengers except those with missing 'Fare' values
", "Shape(rows, columns):", temp1_df.shape )<groupby>
features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str)
 features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features["PoolQC"] = features["PoolQC"].fillna("None")
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
print('Features size:', features.shape )
avg_fares = temp1_df.groupby('Pclass' ).mean()
avg_fares<feature_engineering>
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
validation_df.loc[152, "Fare"] = avg_fares.loc[3, 'Fare']<train_model>
missing = percent_missing(features)
 df_miss = sorted(missing.items() , key=lambda x: x[1], reverse=True)
 print('Percent of missing data')
 df_miss[0:10]
validation_df[validation_df.index == 152]<filter>
testandtrain_df[testandtrain_df.Embarked.isna() ]<count_values>
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics2 = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics2.append(i )
testandtrain_df.Embarked.value_counts() + validation_df.Embarked.value_counts()<feature_engineering>
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))
testandtrain_df.loc[61, "Embarked"] = "S"
testandtrain_df.loc[829, "Embarked"] = "S"<drop_column>
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
for df in [testandtrain_df, validation_df]:
df.drop(columns = ['Cabin', 'Ticket'], inplace = True )<train_model>
features = features.drop(['MasVnrArea', 'OpenPorchSF', 'WoodDeckSF', 'BsmtFinSF1','2ndFlrSF'], axis=1)
testandtrain_df.drop(columns = ['PassengerId'], inplace = True )<feature_engineering>
features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 print('Features size:', features.shape )
for df in [testandtrain_df, validation_df]:
for row in df.index:
df.loc[row, 'Title'] = df.loc[row, 'Name'].split(", ")[1].split(".")[0]<count_unique_values>
final_features = pd.get_dummies(features ).reset_index(drop=True)
 print('Features size:', features.shape)
 final_features.head()
temp2_df = pd.concat([testandtrain_df.copy() , validation_df.copy() ], sort = False)
 print("Number of unique titles(in both dataframes):", temp2_df.Title.nunique())
 print("Number of null titles(in both dataframes):", temp2_df.Title.isna().sum())
 temp2_df.groupby("Title" ).count().Name<feature_engineering>
X = final_features.iloc[:len(y), :]
X_test = final_features.iloc[len(X):, :]
print('Features size for train(X,y)and test(X_test):')
 print('X', X.shape, 'y', y.shape, 'X_test', X_test.shape)
prof_titles = ["Capt", "Col", "Dr", "Major", "Rev"]
other_titles = ["Don", "Dona", "Jonkheer", "Lady", "Mlle", "Mme", "Ms", "Sir", "the Countess"]
for df in [testandtrain_df, validation_df]:
for row in df.index:
if df.loc[row, 'Title'] in prof_titles:
df.loc[row, 'Title'] = "Professional"
elif df.loc[row, 'Title'] in other_titles:
df.loc[row, 'Title'] = "Other" 
<drop_column>
outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers] )
for df in [testandtrain_df, validation_df]:
df.drop(columns = ['Name'], inplace = True )<create_dataframe>
overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.95:
overfit.append(i)
 overfit = list(overfit)
 overfit.append('MSZoning_C(all)')
 X = X.drop(overfit, axis=1 ).copy()
X_test = X_test.drop(overfit, axis=1 ).copy()
print('X', X.shape, 'y', y.shape, 'X_test', X_test.shape )
temp3_df = pd.concat([testandtrain_df.copy() , validation_df.copy() ], sort = False)
 title_ages = temp3_df.groupby("Title" ).Age.mean()
title_ages<feature_engineering>
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y,scoring="neg_mean_squared_error",cv=kfolds))
return(rmse)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
for df in [testandtrain_df, validation_df]:
for row in df.index:
if pd.isna(df.loc[row, 'Age']):
df.loc[row, 'Age'] = title_ages[df.loc[row, 'Title']]<count_missing_values>
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7,
alphas=alphas2,random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7,
alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
models = {'Ridge': ridge,
'Lasso': lasso,
'ElasticNet': elasticnet}
predictions = {}
scores = {}
for name, model in models.items() :
model.fit(X, y)
 predictions[name] = np.expm1(model.predict(X))
score = cv_rmse(model, X=X)
 scores[name] =(score.mean() , score.std())
print("Total null values in both dataframes:", testandtrain_df.isna().sum().sum() + validation_df.isna().sum().sum() )<drop_column>
print('---- Score with CV_RMSLE-----')
 score = cv_rmse(ridge)
 print("Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_rmse(lasso)
 print("Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_rmse(elasticnet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
for df in [testandtrain_df, validation_df]:
df["Family_Onboard"] = df["SibSp"] + df["Parch"]
df.drop(columns = ['SibSp', 'Parch'], inplace = True )<filter>
print('----START Fit----',datetime.now())
 print('Elasticnet')
 elastic_model = elasticnet.fit(X, y)
 print('Lasso')
 lasso_model = lasso.fit(X, y)
 print('Ridge')
 ridge_model = ridge.fit(X, y)
testandtrain_df[testandtrain_df["Fare"] == 0]<filter>
print('Predict submission')
 y_test_r = pd.read_csv(".. /input/sample_submission.csv")
 submission = pd.read_csv(".. /input/sample_submission.csv")
 y_test = np.log1p(y_test_r.iloc[:,1].values)
 submission.iloc[:,1] = np.expm1(blend_models(X_test))
blended_score = rmsle(y_test, blend_models(X_test))
validation_df[validation_df["Fare"] == 0]<train_model>
submission.to_csv("new_submission.csv", index=False)
 print('Save submission', datetime.now() , )