Kaggle competition: Zillow Prize: Zillow’s Home Value Prediction (Zestimate)
<SOS> metric: ZillowMAE Kaggle data source: zillow-prize:-zillow’s-home-value-prediction-(zestimate)<count_duplicates>
import numpy as np
import pandas as pd
import xgboost as xgb
from sklearn.preprocessing import LabelEncoder
import lightgbm as lgb
import gc
from sklearn.linear_model import LinearRegression
import random
import datetime as dt
import matplotlib.pyplot as plt
import seaborn as sb
new_train_data_cabin.duplicated().sum()<count_duplicates>
%matplotlib inline
sb.set_style("whitegrid")
 ints = ['parcelid']
floats = ['basementsqft', 'bathroomcnt', 'bedroomcnt', 'calculatedbathnbr', 'finishedfloor1squarefeet',
'calculatedfinishedsquarefeet', 'finishedsquarefeet12', 'finishedsquarefeet13',
'finishedsquarefeet15', 'finishedsquarefeet50', 'finishedsquarefeet6', 'fireplacecnt',
'fullbathcnt', 'garagecarcnt', 'garagetotalsqft', 'latitude', 'longitude',
'lotsizesquarefeet', 'poolcnt', 'poolsizesum', 'roomcnt', 'threequarterbathnbr', 'unitcnt',
'yardbuildingsqft17', 'yardbuildingsqft26', 'yearbuilt', 'numberofstories',
'structuretaxvaluedollarcnt', 'taxvaluedollarcnt', 'assessmentyear',
'landtaxvaluedollarcnt', 'taxamount', 'taxdelinquencyyear']
objects = ['airconditioningtypeid', 'architecturalstyletypeid', 'buildingclasstypeid',
'buildingqualitytypeid', 'decktypeid', 'fips', 'hashottuborspa', 'heatingorsystemtypeid',
'pooltypeid10', 'pooltypeid2', 'pooltypeid7', 'propertycountylandusecode',
'propertylandusetypeid', 'propertyzoningdesc', 'rawcensustractandblock', 'regionidcity',
'regionidcounty', 'regionidneighborhood', 'regionidzip', 'storytypeid',
'typeconstructiontypeid', 'fireplaceflag', 'taxdelinquencyflag', 'censustractandblock']
feature_dtypes = {col: col_type for type_list, col_type in zip([ints, floats, objects],
['int64', 'float64', 'object'])
 for col in type_list}
data = pd.read_csv('.. /input/properties_2016.csv' , dtype = feature_dtypes)
 data.head()
new_train_data_cabin.drop_duplicates(inplace=True)
 new_train_data_cabin.duplicated().sum()<create_dataframe>
continuous = ['basementsqft', 'finishedfloor1squarefeet', 'calculatedfinishedsquarefeet',
'finishedsquarefeet12', 'finishedsquarefeet13', 'finishedsquarefeet15',
'finishedsquarefeet50', 'finishedsquarefeet6', 'garagetotalsqft', 'latitude',
'longitude', 'lotsizesquarefeet', 'poolsizesum', 'yardbuildingsqft17',
'yardbuildingsqft26', 'yearbuilt', 'structuretaxvaluedollarcnt', 'taxvaluedollarcnt',
'landtaxvaluedollarcnt', 'taxamount']
discrete = ['bathroomcnt', 'bedroomcnt', 'calculatedbathnbr', 'fireplacecnt', 'fullbathcnt',
'garagecarcnt', 'poolcnt', 'roomcnt', 'threequarterbathnbr', 'unitcnt',
'numberofstories', 'assessmentyear', 'taxdelinquencyyear']
print("
Continuous variable plots
")
 for col in continuous:
values = data[col].dropna()
lower = np.percentile(values, 1)
 upper = np.percentile(values, 99)
 fig = plt.figure(figsize=(18,9)) ;
sb.distplot(values[(values>lower)&(values<upper)], color='purple', ax = plt.subplot(121)) ;
plt.suptitle(col, fontsize=16 )
training = new_train_data_cabin.copy()
training<train_model>
print("
Reading data from disk...")
 prop = pd.read_csv('.. /input/properties_2016.csv')
 train = pd.read_csv(".. /input/train_2016_v2.csv")
 XGB_WEIGHT = 0.6415
BASELINE_WEIGHT = 0.0050
XGB1_WEIGHT = 0.8083
BASELINE_PRED = 0.0115
print("
Re-reading properties file...")
 properties = pd.read_csv('.. /input/properties_2016.csv')
 print("
Processing data for XGBoost...")
 for c in properties.columns:
properties[c]=properties[c].fillna(-1)
 if properties[c].dtype == 'object':
lbl = LabelEncoder()
lbl.fit(list(properties[c].values))
properties[c] = lbl.transform(list(properties[c].values))
train_df = train.merge(properties, how='left', on='parcelid')
 train_df["transactiondate"] = pd.to_datetime(train_df["transactiondate"])
 train_df["Month"] = train_df["transactiondate"].dt.month
x_train = train_df.drop(['parcelid', 'logerror','transactiondate'], axis=1)
 x_test = properties.drop(['parcelid'], axis=1)
 x_test["transactiondate"] = '2016-07-01'
x_test["transactiondate"] = pd.to_datetime(x_test["transactiondate"])
 x_test["Month"] = x_test["transactiondate"].dt.month
x_test = x_test.drop(['transactiondate'], axis=1)
 print('Shape train: {}
Shape test: {}'.format(x_train.shape, x_test.shape))
train_df=train_df[ train_df.logerror > -0.4 ]
train_df=train_df[ train_df.logerror < 0.419 ]
x_train=train_df.drop(['parcelid', 'logerror','transactiondate'], axis=1)
 y_train = train_df["logerror"].values.astype(np.float32)
 y_mean = np.mean(y_train)
 print('After removing outliers:')
 print('Shape train: {}
Shape test: {}'.format(x_train.shape, x_test.shape))
print("
Setting up data for XGBoost...")
 xgb_params = {
'eta': 0.037,
'max_depth': 5,
'subsample': 0.80,
'objective': 'reg:linear',
'eval_metric': 'mae',
'lambda': 0.8,
'alpha': 0.4,
'base_score': y_mean,
'silent': 1
}
dtrain = xgb.DMatrix(x_train, y_train)
 dtest = xgb.DMatrix(x_test)
 num_boost_rounds = 250
print("num_boost_rounds="+str(num_boost_rounds))
print("
Training XGBoost...")
 model = xgb.train(dict(xgb_params, silent=1), dtrain, num_boost_round=num_boost_rounds)
 print("
Predicting with XGBoost...")
 xgb_pred1 = model.predict(dtest)
 print("
First XGBoost predictions:")
 print(pd.DataFrame(xgb_pred1 ).head())
 print("
Setting up data for XGBoost...")
 xgb_params = {
'eta': 0.033,
'max_depth': 6,
'subsample': 0.80,
'objective': 'reg:linear',
'eval_metric': 'mae',
'base_score': y_mean,
'silent': 1
}
num_boost_rounds = 150
print("num_boost_rounds="+str(num_boost_rounds))
print("
Training XGBoost again...")
 model = xgb.train(dict(xgb_params, silent=1), dtrain, num_boost_round=num_boost_rounds)
 print("
Predicting with XGBoost again...")
 xgb_pred2 = model.predict(dtest)
 print("
Second XGBoost predictions:")
 print(pd.DataFrame(xgb_pred2 ).head())
 xgb_pred = XGB1_WEIGHT*xgb_pred1 +(1-XGB1_WEIGHT)*xgb_pred2
print("
Combined XGBoost predictions:")
 print(pd.DataFrame(xgb_pred ).head())
 del train_df
del x_train
del x_test
del properties
del dtest
del dtrain
del xgb_pred1
del xgb_pred2
gc.collect()
training.drop('C',axis=1,inplace=True)
 training.drop('Q',axis=1,inplace=True)
 training.drop('S',axis=1,inplace=True)
 training<prepare_x_and_y>
print("
Reading data from disk...")
 prop = pd.read_csv('.. /input/properties_2016.csv')
 train = pd.read_csv(".. /input/train_2016_v2.csv", parse_dates=["transactiondate"])
 properties = pd.read_csv(".. /input/properties_2016.csv")
 submission = pd.read_csv(".. /input/sample_submission.csv")
 OLS_WEIGHT = 0.0856
np.random.seed(17)
 random.seed(17)
 print(len(train),len(properties),len(submission))
def get_features(df):
df["transactiondate"] = pd.to_datetime(df["transactiondate"])
 df["transactiondate_year"] = df["transactiondate"].dt.year
df["transactiondate_month"] = df["transactiondate"].dt.month
df['transactiondate'] = df['transactiondate'].dt.quarter
df = df.fillna(-1.0)
 return df
def MAE(y, ypred):
return np.sum([abs(y[i]-ypred[i])for i in range(len(y)) ])/ len(y)
 train = pd.merge(train, properties, how='left', on='parcelid')
 y = train['logerror'].values
test = pd.merge(submission, properties, how='left', left_on='ParcelId', right_on='parcelid')
 properties = []
exc = [train.columns[c] for c in range(len(train.columns)) if train.dtypes[c] == 'O'] + ['logerror','parcelid']
col = [c for c in train.columns if c not in exc]
train = get_features(train[col])
 test['transactiondate'] = '2016-01-01'
test = get_features(test[col])
 reg = LinearRegression(n_jobs=-1)
 reg.fit(train, y); print('fit...')
 print(MAE(y, reg.predict(train)))
 train = []; y = []
test_dates = ['2016-10-01','2016-11-01','2016-12-01','2017-10-01','2017-11-01','2017-12-01']
test_columns = ['201610','201611','201612','201710','201711','201712']
x_data = training.drop('Survived',axis=1,inplace=False)
 y_data = training[['Survived']]<drop_column>
print("
Reading data from disk...")
 prop = pd.read_csv('.. /input/properties_2016.csv')
 train = pd.read_csv(".. /input/train_2016_v2.csv")
 XGB_WEIGHT = 0.6415
BASELINE_WEIGHT = 0.0050
XGB1_WEIGHT = 0.8083
BASELINE_PRED = 0.0115
print("
Processing data for LightGBM...")
 for c, dtype in zip(prop.columns, prop.dtypes):	
if dtype == np.float64:		
prop[c] = prop[c].astype(np.float32)
 df_train = train.merge(prop, how='left', on='parcelid')
 df_train.fillna(df_train.median() ,inplace = True)
 df_train["transactiondate"] = pd.to_datetime(df_train["transactiondate"])
 df_train["Month"] = df_train["transactiondate"].dt.month
x_train = df_train.drop(['parcelid', 'logerror', 'transactiondate', 'propertyzoningdesc',
'propertycountylandusecode', 'fireplacecnt', 'fireplaceflag'], axis=1)
 y_train = df_train['logerror'].values
print(x_train.shape, y_train.shape)
 train_columns = x_train.columns
for c in x_train.dtypes[x_train.dtypes == object].index.values:
x_train[c] =(x_train[c] == True)
 del df_train; gc.collect()
x_train = x_train.values.astype(np.float32, copy=False)
 d_train = lgb.Dataset(x_train, label=y_train)
 params = {}
params['max_bin'] = 10
params['learning_rate'] = 0.0021
params['boosting_type'] = 'gbdt'
params['objective'] = 'regression'
params['metric'] = 'l1'
params['sub_feature'] = 0.345
params['bagging_fraction'] = 0.85
params['bagging_freq'] = 40
params['num_leaves'] = 512
params['min_data'] = 500
params['min_hessian'] = 0.05
params['verbose'] = 0
params['feature_fraction_seed'] = 2
params['bagging_seed'] = 3
np.random.seed(0)
 random.seed(0)
 print("
Fitting LightGBM model...")
 clf = lgb.train(params, d_train, 430)
 del d_train; gc.collect()
del x_train; gc.collect()
print("
Prepare for LightGBM prediction...")
 print(" Read sample file...")
 sample = pd.read_csv('.. /input/sample_submission.csv')
 print("...")
 sample['parcelid'] = sample['ParcelId']
print(" Merge with property data...")
 df_test = sample.merge(prop, on='parcelid', how='left')
 df_test["transactiondate"] = '2016-07-01'
df_test["transactiondate"] = pd.to_datetime(df_test["transactiondate"])
 df_test["Month"] = df_test["transactiondate"].dt.month
df_test = df_test.drop(['transactiondate'], axis=1)
 print("...")
 del sample, prop; gc.collect()
print("...")
 x_test = df_test[train_columns]
print("...")
 del df_test; gc.collect()
print(" Preparing x_test...")
 for c in x_test.dtypes[x_test.dtypes == object].index.values:
x_test[c] =(x_test[c] == True)
 print("...")
 x_test = x_test.values.astype(np.float32, copy=False)
 print("
Start LightGBM prediction...")
 p_test = clf.predict(x_test)
 del x_test; gc.collect()
print("
Unadjusted LightGBM predictions:")
 print(pd.DataFrame(p_test ).head())
x_data.drop('mean_fare_sex',axis=1,inplace = True )<data_type_conversions>
print("
Combining XGBoost, LightGBM, and baseline predicitons...")
 lgb_weight =(1 - XGB_WEIGHT - BASELINE_WEIGHT)/(1 - OLS_WEIGHT)
 xgb_weight0 = XGB_WEIGHT /(1 - OLS_WEIGHT)
 baseline_weight0 = BASELINE_WEIGHT /(1 - OLS_WEIGHT)
 pred0 = xgb_weight0*xgb_pred + baseline_weight0*BASELINE_PRED + lgb_weight*p_test
print("
Combined XGB/LGB/baseline predictions:")
 print(pd.DataFrame(pred0 ).head())
 print("
Predicting with OLS and combining with XGB/LGB/baseline predicitons:...")
 for i in range(len(test_dates)) :
test['transactiondate'] = test_dates[i]
pred = OLS_WEIGHT*reg.predict(get_features(test)) +(1-OLS_WEIGHT)*pred0
submission[test_columns[i]] = [float(format(x, '.4f')) for x in pred]
print('predict...', i)
 print("
Combined XGB/LGB/baseline/OLS predictions:")
 print(submission.head() )
<drop_column><EOS> 
print("
Writing results to disk...")
 submission.to_csv('TEm07sub{}.csv'.format(datetime.now().strftime('%Y%m%d_%H%M%S')) , index=False)