Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<import_modules>
warnings.filterwarnings('ignore')
 pd.set_option('display.max_rows', None)
 pd.set_option('display.max_columns', None)
 print("Primary Directory:")
 print(os.listdir(".. /input/"))
print("
Files:")
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
optuna.__version__<import_modules>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 def quick_look(df, df_name):
print('{} data contains {} observations across {} columns
'.format(df_name, df.shape[0], df.shape[1]))
display(df.head(5))
quick_look(train, 'Train')
 quick_look(test, 'Test' )
import numpy as np
import pandas as pd
import lightgbm as lgb
import gc
import datetime as dt
from pathlib import Path<define_variables>
print('Only difference in columns between train and test is our target variable: {}'.format(( set(train.columns ).symmetric_difference(set(test.columns)))))
 train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True )
input_path = Path(".. /input/zillow-prize-1/")
 properties_2016_path = input_path / "properties_2016.csv"
train_2016_v2_path = input_path / "train_2016_v2.csv"
train_2017_path = input_path / "train_2017.csv"
properties_2017_path = input_path / "properties_2017.csv"
sample_submission_path = input_path / "sample_submission.csv"<load_from_csv>
train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index)
 train.reset_index(drop=True, inplace=True )
print('Loading data...')
 properties = pd.read_csv(properties_2016_path, low_memory = False)
 train = pd.read_csv(train_2016_v2_path)
 sample_submission = pd.read_csv(sample_submission_path, low_memory = False)
 train = pd.merge(train, properties, how = 'left', on = 'parcelid')
 test = pd.merge(sample_submission[['ParcelId']], properties.rename(columns = {'parcelid': 'ParcelId'}),
how = 'left', on = 'ParcelId')
 del properties
gc.collect() ;<data_type_conversions>
complete_df = pd.concat(( train.iloc[:,0:-1],test.iloc[:,0:len(test.columns)]))
def data_quality(df):
print('Data Quality for Complete dataframe')
 df_temp = pd.DataFrame(index=df.columns, columns=['Missing_Count', 'Unique_Values', 'Scarcity', 'Datatype'])
 df_temp['Missing_Count'] = df.isnull().sum()
df_temp['Unique_Values'] = [df[i].nunique() for i in df.columns]
df_temp['Scarcity'] = round(((df.isnull().sum())/(df.shape[0])*100),2)
 df_temp['Datatype'] = [df[i].dtype for i in df.columns]
return df_temp.head(len(df.columns)).sort_values(by=['Missing_Count'], ascending=False)
 dq_df = data_quality(complete_df)
 display(dq_df )
print('Memory usage reduction...')
 train[['latitude', 'longitude']] /= 1e6
test[['latitude', 'longitude']] /= 1e6
train['censustractandblock'] /= 1e12
test['censustractandblock'] /= 1e12
for column in test.columns:
if test[column].dtype == int:
test[column] = test[column].astype(np.int32)
 if test[column].dtype == float:
test[column] = test[column].astype(np.float32 )<data_type_conversions>
complete_df.drop(['GarageCars', 'GarageYrBlt'], axis=1, inplace=True )
print('Feature engineering...')
 train['month'] = pd.to_datetime(train['transactiondate'] ).dt.month
train = train.drop('transactiondate', axis = 1)
 non_number_columns = train.dtypes[train.dtypes == object].index.values
for column in non_number_columns:
train_test = pd.concat([train[column], test[column]], axis = 0)
 encoder = LabelEncoder().fit(train_test.astype(str))
train[column] = encoder.transform(train[column].astype(str)).astype(np.int32)
 test[column] = encoder.transform(test[column].astype(str)).astype(np.int32)
 feature_names = train.columns[2:]
feature_names = [feature for feature in feature_names if feature != 'month']
month_avgs = train.groupby('month' ).agg(['mean'])['logerror', 'mean'].values - train['logerror'].mean()
month_model = LinearRegression().fit(np.arange(4, 13, 1 ).reshape(-1, 1),
month_avgs[3:].reshape(-1, 1))<train_model>
missing_cols = [i for i in dq_df[dq_df['Scarcity'] > 0].index if i not in {'GarageCars', 'GarageYrBlt'}]
other_cols = ['SaleType', 'Exterior1st', 'Exterior2nd', 'Functional', 'MSZoning']
for i in missing_cols:
if complete_df[i].dtypes != 'object':
complete_df[i] = complete_df[i].fillna(complete_df[i].mean())
 elif i in other_cols:
complete_df[i] = complete_df[i].fillna('Other')
 else:
complete_df[i] = complete_df[i].fillna('None' )
print('Preparing arrays and throwing out outliers...')
 X_train = train[feature_names].values
y_train = train.iloc[:, 1].values
X_test = test[feature_names].values
del test
gc.collect() ;
month_values = train['month'].values
X_train = np.hstack([X_train, month_model.predict(month_values.reshape(-1, 1)) ])
 X_train = X_train[np.abs(y_train)< 0.4, :]
y_train = y_train[np.abs(y_train)< 0.4]
print('Training LGBM model...')
 ltrain = lgb.Dataset(X_train, label = y_train)
 params = {}
params['metric'] = 'mae'
params['max_depth'] = 100
params['num_leaves'] = 32
params['feature_fraction'] =.85
params['bagging_fraction'] =.95
params['bagging_freq'] = 8
params['learning_rate'] = 0.0025
params['verbosity'] = 0
lgb_model = lgb.train(params, ltrain, verbose_eval=0, num_boost_round=2930 )<prepare_output>
num_cols = [i for i in num_cols if i not in {'GarageCars', 'GarageYrBlt', 'SalePrice', 'Id'}]
skewed_data = complete_df[num_cols].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skewed_data[skewed_data >.75]
display(high_skew.head(25))
high_skew_feats = high_skew.index
complete_df[high_skew_feats] = np.log1p(complete_df[high_skew_feats] )
print('Making predictions and praying for good results...')
 X_test = np.hstack([X_test, np.zeros(( X_test.shape[0], 1)) ])
 folds = 20
n = int(X_test.shape[0] / folds)
 for j in range(folds):
results = pd.DataFrame()
if j < folds - 1:
X_test_ = X_test[j*n:(j+1)*n, :]
results['ParcelId'] = sample_submission['ParcelId'].iloc[j*n:(j+1)*n]
else:
X_test_ = X_test[j*n: , :]
results['ParcelId'] = sample_submission['ParcelId'].iloc[j*n: ]
for month in [10, 11, 12]:
X_test_[:, -1] = month_avgs[month - 1]
assert X_test_.shape[1] == X_test.shape[1]
y_pred = lgb_model.predict(X_test_)
 results['2016' + str(month)] = y_pred
results['2017' + str(month)] = y_pred
if j == 0:
results_ = results.copy()
else:
results_ = pd.concat([results_, results], axis = 0)
 print('{}% completed'.format(round(100*(j+1)/folds)) )<save_to_csv>
complete_df['TotalSF']=(complete_df['TotalBsmtSF'] +
complete_df['1stFlrSF'] +
complete_df['2ndFlrSF'])
 complete_df['TotBathrooms'] =(( complete_df['FullBath'])+ 
(complete_df['HalfBath'] *.5)+ 
(complete_df['BsmtFullBath'])+ 
(complete_df['BsmtHalfBath'] *.5))
complete_df['TotPorchSF'] =(complete_df['OpenPorchSF'] +
complete_df['EnclosedPorch'] +
complete_df['3SsnPorch'] +
complete_df['ScreenPorch'] +
complete_df['WoodDeckSF'])
 complete_df['BedToTotRmsRatio'] =(complete_df['BedroomAbvGr'] / complete_df['TotRmsAbvGrd'] )
print('Saving predictions...')
 results = results_[sample_submission.columns]
assert results.shape == sample_submission.shape
results.to_csv('submission.csv', index = False, float_format = '%.6f')
 print('Done!' )<set_options>
cat_cols_convert = ['Street', 'BldgType', 'HouseStyle', 'Electrical',
'KitchenQual', 'Functional', 'FireplaceQu', 'GarageQual',
'GarageCond', 'PavedDrive', 'PoolQC']
le = LabelEncoder()
complete_df[cat_cols_convert] = complete_df[cat_cols_convert].apply(lambda col: le.fit_transform(col))
print('Converted Encoded Features:
')
 display(complete_df[cat_cols_convert].head(5))
num_cols = [i for i in num_cols if i not in {'MSSubClass', 'MoSold', 'YrSold'}]
scaler = RobustScaler()
complete_df[num_cols] = scaler.fit_transform(complete_df[num_cols])
 print('
Converted Scaled Numeric features:')
 display(complete_df[num_cols].head(5))
cat_cols = [i for i in cat_cols if i not in {'Street', 'BldgType', 'HouseStyle', 'Electrical','KitchenQual', 'Functional', 'FireplaceQU', 'GarageQual','GarageCond', 'PavedDrive', 'PoolQC'}]
changed_cols = ['MSSubClass', 'MoSold', 'YrSold']
cat_cols = cat_cols + changed_cols
complete_df[cat_cols] = complete_df[cat_cols].astype('category')
 complete_df = pd.get_dummies(complete_df)
 print('
Model ready dataframe after label encoding, robust scaling, and one hot encoding:')
 display(complete_df.head(10))
%matplotlib inline<load_from_csv>
y = train.SalePrice
X_train = complete_df.iloc[:len(y), :]
X_test = complete_df.iloc[len(y):, :]
print('Loading Properties...')
 properties2016 = pd.read_csv('.. /input/properties_2016.csv', low_memory = False)
 properties2017 = pd.read_csv('.. /input/properties_2017.csv', low_memory = False)
 print('Loading Train...')
 train2016 = pd.read_csv('.. /input/train_2016_v2.csv', parse_dates=['transactiondate'], low_memory=False)
 train2017 = pd.read_csv('.. /input/train_2017.csv', parse_dates=['transactiondate'], low_memory=False)
 print('Loading Sample...')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv', low_memory=False )<feature_engineering>
def run_model(model):
result = np.sqrt(-cross_val_score(model, X_train, y, scoring="neg_mean_squared_error", cv = 5))
return(result )
def add_date_features(df):
df["transaction_year"] = df["transactiondate"].dt.year
df["transaction_month"] = df["transactiondate"].dt.month
df["transaction_day"] = df["transactiondate"].dt.day
df["transaction_quarter"] = df["transactiondate"].dt.quarter
df.drop(["transactiondate"], inplace=True, axis=1)
 return df<merge>
model_ridge = Ridge(alpha=10 ).fit(X_train,y)
 ridge_preds = np.expm1(model_ridge.predict(X_test))
<count_missing_values><EOS> 
final = pd.DataFrame()
final['Id'] = test_ID
final['SalePrice'] = ridge_preds
final.to_csv('submission.csv',index=False )