Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_from_csv>
warnings.filterwarnings('ignore' )
data_train = pd.read_csv('.. /input/train.csv')
 data_train.head()<load_from_csv>
df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv',index_col='Id')
 df_val = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv',index_col='Id')
 y=np.log1p(df.SalePrice)
 df=df.drop(['SalePrice'],axis=1)
 print('shape of df:', df.shape)
 print('shape of df_val:', df_val.shape )
data_test = pd.read_csv('.. /input/test.csv')
 data_test.head()<concatenate>
def remove_nanish(nan_thresh,df,df_val):
X=df
mask1=X.isnull().sum() /len(X)<nan_thresh
mask2=df_val.isnull().sum() /len(df_val)<nan_thresh
keeps=X.columns[mask1 & mask2]
X=X[keeps]
X_valid=df_val[keeps]
return X, X_valid
df = data_train.append(data_test, sort = True)
 df.shape<set_options>
def impute_nums(num_X_train,num_X_test,num_X_valid):
SI=SimpleImputer(strategy='mean')
 num_X_train = pd.DataFrame(SI.fit_transform(num_X_train),index=num_X_train.index, columns=num_X_train.columns)
 num_X_test = pd.DataFrame(SI.transform(num_X_test),index=num_X_test.index, columns=num_X_test.columns)
 num_X_valid = pd.DataFrame(SI.transform(num_X_valid),index=num_X_valid.index, columns=num_X_valid.columns)
 return num_X_train, num_X_test, num_X_valid
%matplotlib inline
plt.style.use('bmh')
 plt.rc('font', family='DejaVu Sans', size=13 )<count_values>
def scale(num_X_train,num_X_test,num_X_valid):
SS=StandardScaler()
num_X_train = pd.DataFrame(SS.fit_transform(num_X_train),index=num_X_train.index, columns=num_X_train.columns)
 num_X_test = pd.DataFrame(SS.transform(num_X_test),index=num_X_test.index, columns=num_X_test.columns)
 num_X_valid = pd.DataFrame(SS.transform(num_X_valid),index=num_X_valid.index, columns=num_X_valid.columns)
 return num_X_train, num_X_test, num_X_valid
cat_list = ['Cabin', 'Embarked', 'Name', 'Parch', 'Pclass', 'Sex', 'SibSp', 'Ticket']
for n, i in enumerate(cat_list):
cat_num = len(df[i].value_counts().index)
 print('The feature "%s" has %d values.' %(i, cat_num))<count_missing_values>
def impute_encode(cat_X_train,cat_X_test,cat_X_valid):
im=SimpleImputer(strategy='most_frequent')
 cat_X_train = pd.DataFrame(im.fit_transform(cat_X_train),index=cat_X_train.index, columns=cat_X_train.columns)
 cat_X_test = pd.DataFrame(im.transform(cat_X_test),index=cat_X_test.index, columns=cat_X_test.columns)
 cat_X_valid = pd.DataFrame(im.transform(cat_X_valid),index=cat_X_valid.index, columns=cat_X_valid.columns)
 OH = OneHotEncoder(handle_unknown='ignore',sparse=False)
 cat_X_train = pd.DataFrame(OH.fit_transform(cat_X_train),index=cat_X_train.index)
 cat_X_test = pd.DataFrame(OH.transform(cat_X_test),index=cat_X_test.index)
 cat_X_valid = pd.DataFrame(OH.transform(cat_X_valid),index=cat_X_valid.index)
 return cat_X_train, cat_X_test, cat_X_valid
df.isnull().sum()<filter>
def preprocess(X,df_val):
nan_thresh=0.0450
skew_thresh=40
target_corr=0.01
X, X_valid=remove_nanish(nan_thresh,df,df_val)
 categorical_cols = [cname for cname in X.columns if X[cname].dtype == "object" ]
numerical_cols = [cname for cname in X.columns if X[cname].dtype in ['int64', 'float64']]
cols=categorical_cols+numerical_cols
X = X[cols]
X_valid= X_valid[cols]
X_train, X_test, y_train, y_test =train_test_split(X,y,test_size=0.2)
 num_X_train = X_train.drop(categorical_cols, axis=1)
 num_X_test = X_test.drop(categorical_cols, axis=1)
 num_X_valid = X_valid.drop(categorical_cols, axis=1)
 num_X_train, num_X_test, num_X_valid= impute_nums(num_X_train, num_X_test, num_X_valid)
 skewed=[col for col in num_X_train.columns if abs(num_X_train[col].skew(axis=0)) >skew_thresh]
for col in skewed:
num_X_train[col]=np.log1p(num_X_train[col])
 num_X_test[col]=np.log1p(num_X_test[col])
 num_X_valid[col]=np.log1p(num_X_valid[col])
 num_X_train, num_X_test, num_X_valid= scale(num_X_train, num_X_test, num_X_valid)
 for col in numerical_cols:
if abs(y.corr(num_X_train[col])) < target_corr:
num_X_train.drop([col],axis='columns',inplace=True)
 num_X_test.drop([col],axis='columns',inplace=True)
 num_X_valid.drop([col],axis='columns',inplace=True)
 numerical_cols = [cname for cname in num_X_train.columns if num_X_train[cname].dtype in ['int64', 'float64']]
cat_X_train = X_train[categorical_cols]
cat_X_test = X_test[categorical_cols]
cat_X_valid = X_valid[categorical_cols]
cat_X_train,cat_X_test,cat_X_valid = impute_encode(cat_X_train,cat_X_test,cat_X_valid)
 X_train = pd.concat([num_X_train, cat_X_train], axis=1,join='inner')
 X_test = pd.concat([num_X_test, cat_X_test ], axis=1,join='inner')
 X_valid = pd.concat([num_X_valid, cat_X_valid ], axis=1,join='inner')
 return X_train, X_test, X_valid, y_train, y_test, numerical_cols
df[df.Fare.isnull() ]<filter>
X_train, X_test, X_valid, y_train, y_test, numerical_cols = preprocess(df,df_val )
df[(df.Pclass == 3)&(df.Age > 60)&(df.Sex == 'male')]<feature_engineering>
best_lasso=Lasso(alpha=0.00045)
 best_lasso.fit(X_train,y_train)
 test_preds=best_lasso.predict(X_test)
 test_rmse=np.sqrt(mean_squared_error(y_test,test_preds))
train_preds=best_lasso.predict(X_train)
 train_rmse=np.sqrt(mean_squared_error(y_train,train_preds))
print(f'train error {train_rmse}')
 print(f'test error {test_rmse}' )
fare_mean = df[(df.Pclass == 3)&(df.Age > 60)&(df.Sex == 'male')].Fare.mean()
df.loc[df.PassengerId == 1044, 'Fare'] = fare_mean
df[df.PassengerId == 1044]<filter>
valid_preds=np.expm1(pd.Series(best_lasso.predict(X_valid),index=X_valid.index))
example=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv',index_col='Id')
 example['SalePrice']=valid_preds
example.to_csv('submission.csv',index='Id' )