Kaggle competition: Zillow Prize: Zillow’s Home Value Prediction (Zestimate)
<SOS> metric: ZillowMAE Kaggle data source: zillow-prize:-zillow’s-home-value-prediction-(zestimate)<count_missing_values>
FUDGE_FACTOR = 1.1200
XGB_WEIGHT = 0.6200
BASELINE_WEIGHT = 0.0100
OLS_WEIGHT = 0.0620
NN_WEIGHT = 0.0800
XGB1_WEIGHT = 0.8000
BASELINE_PRED = 0.0115
test_data['Cabin'].isnull().sum()<count_missing_values>
import numpy as np
import pandas as pd
import xgboost as xgb
import random
import lightgbm as lgb
import datetime as dt
import matplotlib.pyplot as plt
import seaborn as sns
import gc
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score
train_data['Embarked'].isnull().sum()<count_missing_values>
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout, BatchNormalization
from keras.layers.advanced_activations import PReLU
from keras.layers.noise import GaussianDropout
from keras.optimizers import Adam
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import Imputer
test_data['Embarked'].isnull().sum()<count_values>
train_2017 = pd.read_csv('.. /input/train_2017.csv',parse_dates = ["transactiondate"])
 train_2016 = pd.read_csv('.. /input/train_2016_v2.csv',parse_dates = ["transactiondate"])
 frames = [train_2016,train_2017]
train = pd.concat(frames)
 properties = pd.read_csv('.. /input/properties_2017.csv')
 test = pd.read_csv('.. /input/sample_submission.csv')
 test = test.rename(columns = {'ParcelId': 'parcelid'} )
train_data['Cabin'].value_counts().to_frame()<count_values>
print("Training Size:" + str(train.shape))
print("Property Size:" + str(properties.shape))
print("Sample Size:" + str(test.shape))
test_data['Cabin'].value_counts().to_frame()<count_values>
print("
Processing data for LightGBM...")
 for c, dtype in zip(properties.columns, properties.dtypes):
if dtype == np.float64:
properties[c] = properties[c].astype(np.float32)
 df_train = train.merge(properties, how='left', on='parcelid')
 df_train.fillna(df_train.median() ,inplace = True)
 x_train = df_train.drop(['parcelid', 'logerror', 'transactiondate', 'propertyzoningdesc',
'propertycountylandusecode', 'fireplacecnt', 'fireplaceflag'], axis=1)
 y_train = df_train['logerror'].values
print(x_train.shape, y_train.shape )
train_data['Embarked'].value_counts().to_frame()<create_dataframe>
train_columns = x_train.columns
for c in x_train.dtypes[x_train.dtypes == object].index.values:
x_train[c] =(x_train[c] == True)
 del df_train; gc.collect()
x_train = x_train.values.astype(np.float32, copy=False)
 d_train = lgb.Dataset(x_train, label=y_train )
new_train_data = train_data.copy()<drop_column>
params = {}
params['max_bin'] = 10
params['learning_rate'] = 0.0021
params['boosting_type'] = 'gbdt'
params['objective'] = 'regression'
params['metric'] = 'l1'
params['sub_feature'] = 0.345
params['bagging_fraction'] = 0.85
params['bagging_freq'] = 40
params['num_leaves'] = 512
params['min_data'] = 500
params['min_hessian'] = 0.05
params['verbose'] = 0
params['feature_fraction_seed'] = 2
params['bagging_seed'] = 3
np.random.seed(0)
 random.seed(0 )
new_train_data.drop('Name',axis=1,inplace=True )<feature_engineering>
print("
Fitting LightGBM model...")
 clf = lgb.train(params, d_train, 430 )
new_train_data['Embarked'] = new_train_data['Embarked'].replace(np.NAN,"S")
 new_train_data['Embarked'].isnull().sum()<categorify>
del d_train; gc.collect()
del x_train; gc.collect()
new_train_data['Sex'] = pd.get_dummies(new_train_data['Sex'] )<categorify>
print("
Prepare for LightGBM prediction...")
 print(" Read sample file...")
 sample = pd.read_csv('.. /input/sample_submission.csv')
 print("...")
 sample['parcelid'] = sample['ParcelId']
print(" Merge with property data...")
 df_test = sample.merge(properties, on='parcelid', how='left')
 print("...")
 del sample, properties; gc.collect()
print("...")
 x_test = df_test[train_columns]
print("...")
 del df_test; gc.collect()
print(" Preparing x_test...")
 for c in x_test.dtypes[x_test.dtypes == object].index.values:
x_test[c] =(x_test[c] == True)
 print("...")
 x_test = x_test.values.astype(np.float32, copy=False)
 print("
Start LightGBM prediction...")
 p_test = clf.predict(x_test)
 del x_test; gc.collect()
print("
Unadjusted LightGBM predictions:")
 print(pd.DataFrame(p_test ).head())
lb = LabelEncoder()
new_train_data['Embarked'] = lb.fit_transform(new_train_data['Embarked'])
 onehotencode = OneHotEncoder(handle_unknown='ignore')
 encf = pd.DataFrame(onehotencode.fit_transform(new_train_data[['Embarked']] ).toarray())
 new_train_data = new_train_data.join(encf)
 new_train_data.head()<rename_columns>
print("
Re-reading properties file...")
 properties = pd.read_csv('.. /input/properties_2017.csv' )
new_train_data.rename(columns={0:"C",1:"Q",2:"S"},inplace=True )<filter>
print("
Processing data for XGBoost...")
 for c in properties.columns:
properties[c]=properties[c].fillna(-1)
 if properties[c].dtype == 'object':
lbl = LabelEncoder()
lbl.fit(list(properties[c].values))
properties[c] = lbl.transform(list(properties[c].values))
train_df = train.merge(properties, how='left', on='parcelid')
 x_train = train_df.drop(['parcelid', 'logerror','transactiondate'], axis=1)
 x_test = properties.drop(['parcelid'], axis=1 )
new_train_data = new_train_data[new_train_data.Fare > 0]
new_train_data = new_train_data[new_train_data.Fare <= 300]<compute_test_metric>
train_df=train_df[ train_df.logerror > -0.4 ]
train_df=train_df[ train_df.logerror < 0.419 ]
x_train=train_df.drop(['parcelid', 'logerror','transactiondate'], axis=1)
 y_train = train_df["logerror"].values.astype(np.float32)
 y_mean = np.mean(y_train)
 print('After removing outliers:')
 print('Shape train: {}
Shape test: {}'.format(x_train.shape, x_test.shape))
missingno.matrix(test_data )<count_values>
print("
Setting up data for XGBoost...")
 xgb_params = {
'eta': 0.037,
'max_depth': 5,
'subsample': 0.80,
'objective': 'reg:linear',
'eval_metric': 'mae',
'lambda': 0.8,
'alpha': 0.4,
'base_score': y_mean,
'silent': 1
}
new_train_data['Cabin'].value_counts()<count_values>
dtrain = xgb.DMatrix(x_train, y_train)
 dtest = xgb.DMatrix(x_test)
 num_boost_rounds = 250
print("num_boost_rounds="+str(num_boost_rounds))
print("
Training XGBoost...")
 model = xgb.train(dict(xgb_params, silent=1), dtrain, num_boost_round=num_boost_rounds)
 print("
Predicting with XGBoost...")
 xgb_pred1 = model.predict(dtest)
 print("
First XGBoost predictions:")
 print(pd.DataFrame(xgb_pred1 ).head() )
test_data['Cabin'].value_counts()<create_dataframe>
print("
Setting up data for XGBoost...")
 xgb_params = {
'eta': 0.033,
'max_depth': 6,
'subsample': 0.80,
'objective': 'reg:linear',
'eval_metric': 'mae',
'base_score': y_mean,
'silent': 1
}
num_boost_rounds = 150
print("num_boost_rounds="+str(num_boost_rounds))
print("
Training XGBoost again...")
 model = xgb.train(dict(xgb_params, silent=1), dtrain, num_boost_round=num_boost_rounds)
 print("
Predicting with XGBoost again...")
 xgb_pred2 = model.predict(dtest)
 print("
Second XGBoost predictions:")
 print(pd.DataFrame(xgb_pred2 ).head() )
new_train_data_cabin = new_train_data.copy()<create_dataframe>
xgb_pred = XGB1_WEIGHT*xgb_pred1 +(1-XGB1_WEIGHT)*xgb_pred2
print("
Combined XGBoost predictions:")
 print(pd.DataFrame(xgb_pred ).head())
 del train_df
del x_train
del x_test
del properties
del dtest
del dtrain
del xgb_pred1
del xgb_pred2
gc.collect()
new_test_data = test_data.copy()<data_type_conversions>
print("
Processing data for Neural Network...")
 print('
Loading train, prop and sample data...')
 train_2017 = pd.read_csv('.. /input/train_2017.csv',parse_dates = ["transactiondate"])
 train_2016 = pd.read_csv('.. /input/train_2016_v2.csv',parse_dates = ["transactiondate"])
 frames = [train_2016,train_2017]
train = pd.concat(frames)
 prop = pd.read_csv('.. /input/properties_2017.csv')
 sample = pd.read_csv('.. /input/sample_submission.csv' )
new_train_data_cabin['Cabin_type'] = new_train_data_cabin['Cabin'].str.split(r'[0-9]' ).str[0].tolist()
new_test_data['Cabin_type'] = new_test_data['Cabin'].str.split(r'[0-9]' ).str[0].tolist()<count_values>
print('Fitting Label Encoder on properties...')
 for c in prop.columns:
prop[c]=prop[c].fillna(-1)
 if prop[c].dtype == 'object':
lbl = LabelEncoder()
lbl.fit(list(prop[c].values))
prop[c] = lbl.transform(list(prop[c].values))
new_train_data_cabin['Cabin_type'].value_counts()<count_values>
print('Creating training set...')
 df_train = train.merge(prop, how='left', on='parcelid')
 df_train["transactiondate"] = pd.to_datetime(df_train["transactiondate"])
 df_train["transactiondate_year"] = df_train["transactiondate"].dt.year
df_train["transactiondate_month"] = df_train["transactiondate"].dt.month
df_train['transactiondate_quarter'] = df_train['transactiondate'].dt.quarter
df_train["transactiondate"] = df_train["transactiondate"].dt.day
new_test_data['Cabin_type'].value_counts()<categorify>
print('Filling NA/NaN values...')
 df_train.fillna(-1.0 )
lb1 = LabelEncoder()
new_train_data_cabin['Cabin_type'] = lb1.fit_transform(new_train_data_cabin['Cabin_type'].astype(str))
new_test_data['Cabin_type'] = lb1.fit_transform(new_test_data['Cabin_type'].astype(str))<count_values>
print('Creating x_train and y_train from df_train...')
 x_train = df_train.drop(['parcelid', 'logerror', 'transactiondate', 'propertyzoningdesc', 'propertycountylandusecode','fireplacecnt', 'fireplaceflag'], axis=1)
 y_train = df_train["logerror"]
new_train_data_cabin['Cabin_type'].value_counts()<count_values>
y_mean = np.mean(y_train)
 print(x_train.shape, y_train.shape)
 train_columns = x_train.columns
for c in x_train.dtypes[x_train.dtypes == object].index.values:
x_train[c] =(x_train[c] == True )
new_train_data_cabin['Cabin_type'].value_counts()<categorify>
print('Creating df_test...')
 sample['parcelid'] = sample['ParcelId']
print("Merging Sample with property data...")
 df_test = sample.merge(prop, on='parcelid', how='left')
 df_test["transactiondate"] = pd.to_datetime('2016-11-15')
 df_test["transactiondate_year"] = df_test["transactiondate"].dt.year
df_test["transactiondate_month"] = df_test["transactiondate"].dt.month
df_test['transactiondate_quarter'] = df_test['transactiondate'].dt.quarter
df_test["transactiondate"] = df_test["transactiondate"].dt.day
x_test = df_test[train_columns]
new_train_data_cabin['Cabin_type'] = new_train_data_cabin['Cabin_type'].replace(0,11)
 new_train_data_cabin['Cabin_type'] = new_train_data_cabin['Cabin_type'].replace(10,0)
 new_train_data_cabin['Cabin_type'] = new_train_data_cabin['Cabin_type'].replace(11,10 )<feature_engineering>
print("
Preprocessing neural network data...")
 imputer= Imputer()
imputer.fit(x_train.iloc[:, :])
 x_train = imputer.transform(x_train.iloc[:, :])
 imputer.fit(x_test.iloc[:, :])
 x_test = imputer.transform(x_test.iloc[:, :])
 sc = StandardScaler()
x_train = sc.fit_transform(x_train)
 x_test = sc.transform(x_test)
 len_x=int(x_train.shape[1])
 print("len_x is:",len_x )
new_test_data['Cabin_type'] = new_test_data['Cabin_type'].replace(0,11)
 new_test_data['Cabin_type'] = new_test_data['Cabin_type'].replace(10,0)
 new_test_data['Cabin_type'] = new_test_data['Cabin_type'].replace(11,10 )<count_values>
print("
Setting up neural network model...")
 nn = Sequential()
nn.add(Dense(units = 400 , kernel_initializer = 'normal', input_dim = len_x))
nn.add(PReLU())
 nn.add(Dropout (.4))
nn.add(Dense(units = 160 , kernel_initializer = 'normal'))
nn.add(PReLU())
 nn.add(BatchNormalization())
 nn.add(Dropout (.6))
nn.add(Dense(units = 64 , kernel_initializer = 'normal'))
nn.add(PReLU())
 nn.add(BatchNormalization())
 nn.add(Dropout (.5))
nn.add(Dense(units = 26, kernel_initializer = 'normal'))
nn.add(PReLU())
 nn.add(BatchNormalization())
 nn.add(Dropout (.6))
nn.add(Dense(1, kernel_initializer='normal'))
nn.compile(loss='mae', optimizer=Adam(lr=4e-3, decay=1e-4))
new_train_data_cabin['Cabin_type'].value_counts()<count_values>
print("
Fitting neural network model...")
 nn.fit(np.array(x_train), np.array(y_train), batch_size = 32, epochs = 70, verbose=2)
 print("
Predicting with neural network model...")
 y_pred_ann = nn.predict(x_test )
new_test_data['Cabin_type'].value_counts()<feature_engineering>
del train
del prop
del sample
del x_train
del x_test
del df_train
del df_test
del y_pred_ann
gc.collect()
new_train_data_cabin['Family'] = new_train_data_cabin['SibSp'] + new_train_data_cabin['Parch'] + 1
new_train_data_cabin<feature_engineering>
np.random.seed(17)
 random.seed(17)
 print("
Processing data for OLS...")
 train_2017 = pd.read_csv('.. /input/train_2017.csv',parse_dates = ["transactiondate"])
 train_2016 = pd.read_csv('.. /input/train_2016_v2.csv',parse_dates = ["transactiondate"])
 frames = [train_2016,train_2017]
train = pd.concat(frames)
 properties = pd.read_csv(".. /input/properties_2017.csv")
 submission = pd.read_csv(".. /input/sample_submission.csv")
 print(len(train),len(properties),len(submission))
def get_features(df):
df["transactiondate"] = pd.to_datetime(df["transactiondate"])
 df["transactiondate_year"] = df["transactiondate"].dt.year
df["transactiondate_month"] = df["transactiondate"].dt.month
df['transactiondate'] = df['transactiondate'].dt.quarter
df = df.fillna(-1.0)
 return df
new_test_data['Family'] = new_test_data['SibSp'] + new_test_data['Parch'] + 1
new_test_data<categorify>
def MAE(y, ypred):
return np.sum([abs(y[i]-ypred[i])for i in range(len(y)) ])/ len(y )
new_test_data.drop('Name',axis=1,inplace=True)
 new_test_data['Sex'] = pd.get_dummies(new_test_data['Sex'])
 new_test_data<concatenate>
train = pd.merge(train, properties, how='left', on='parcelid')
 y = train['logerror'].values
test = pd.merge(submission, properties, how='left', left_on='ParcelId', right_on='parcelid')
 properties = []
exc = [train.columns[c] for c in range(len(train.columns)) if train.dtypes[c] == 'O'] + ['logerror','parcelid']
col = [c for c in train.columns if c not in exc]
len(set(test_data.Ticket)- set(test_data.Ticket ).intersection(set(new_train_data_cabin.Ticket))),len(set(test_data.Ticket)) ,len(test_data )<merge>
train = get_features(train[col])
 test['transactiondate'] = '2016-01-01'
test = get_features(test[col] )
group = new_train_data_cabin.groupby(['Sex'] ).agg({'Fare':['mean']})
 group.columns = ['mean_fare_sex']
group.reset_index(inplace=True)
 new_train_data_cabin = pd.merge(new_train_data_cabin,group,on=['Sex'], how='left')
 new_train_data_cabin<merge>
print("
Fitting OLS...")
 reg = LinearRegression(n_jobs=-1)
 reg.fit(train, y); print('fit...')
 print(MAE(y, reg.predict(train)))
 train = []; y = []
test_dates = ['2016-10-01','2016-11-01','2016-12-01','2017-10-01','2017-11-01','2017-12-01']
test_columns = ['201610','201611','201612','201710','201711','201712']
group = new_test_data.groupby(['Sex'] ).agg({'Fare':['mean']})
 group.columns = ['mean_fare_sex']
group.reset_index(inplace=True)
 new_test_data = pd.merge(new_test_data,group,on=['Sex'], how='left')
 new_test_data<drop_column>
print("
Combining XGBoost, LightGBM, NN, and baseline predicitons...")
 lgb_weight = 1 - XGB_WEIGHT - BASELINE_WEIGHT - NN_WEIGHT - OLS_WEIGHT
lgb_weight0 = lgb_weight /(1 - OLS_WEIGHT)
 xgb_weight0 = XGB_WEIGHT /(1 - OLS_WEIGHT)
 baseline_weight0 = BASELINE_WEIGHT /(1 - OLS_WEIGHT)
 nn_weight0 = NN_WEIGHT /(1 - OLS_WEIGHT)
 pred0 = 0
pred0 += xgb_weight0*xgb_pred
pred0 += baseline_weight0*BASELINE_PRED
pred0 += lgb_weight0*p_test
pred0 += nn_weight0*nn_pred
new_train_data_cabin.drop('Ticket',axis=1, inplace=True)
 new_train_data_cabin<drop_column>
print("
Predicting with OLS and combining with XGB/LGB/NN/baseline predicitons:...")
 for i in range(len(test_dates)) :
test['transactiondate'] = test_dates[i]
pred = FUDGE_FACTOR *(OLS_WEIGHT*reg.predict(get_features(test)) +(1-OLS_WEIGHT)*pred0)
 submission[test_columns[i]] = [float(format(x, '.4f')) for x in pred]
print('predict...', i)
 print("
Writing results to disk...")
 submission.to_csv('1003_try.csv', index=False , float_format='%.4f')
 print("
Finished..." )
new_test_data.drop('Ticket',axis=1, inplace=True)
 new_test_data<drop_column>
missing_df = properties.isnull().sum(axis=0 ).reset_index()
missing_df.columns = ['column_name','missing_count']
missing_df['missing_ratio'] = missing_df['missing_count'] / properties.shape[0]
missing_filtered = missing_df.loc[missing_df['missing_ratio']>0.99]
eliminate_list = missing_filtered['column_name'].values
properties = properties.drop(eliminate_list,axis=1 )
new_train_data_cabin.drop('PassengerId',axis=1,inplace=True )<drop_column>
def convert_datatype(dataframe):
for c, dtype in zip(dataframe.columns, dataframe.dtypes):
if dtype == np.float64:
dataframe[c] = dataframe[c].astype(np.float32)
 if dtype == np.int64:
dataframe[c] = dataframe[c].astype(np.int32)
 convert_datatype(properties)
 convert_datatype(test )
new_test_data.drop('PassengerId',axis=1,inplace=True )<count_values>
properties['living_area_prop'] = properties['calculatedfinishedsquarefeet'] / \
properties['lotsizesquarefeet']
properties['value_ratio'] = properties['taxvaluedollarcnt'] / properties['taxamount']
properties['value_prop'] = properties['structuretaxvaluedollarcnt'] /\
properties['landtaxvaluedollarcnt']
new_train_data_cabin['Pclass'].value_counts()<drop_column>
df_train = train.merge(properties, how='left', on='parcelid')
 df_test = test.merge(properties, how='left', on='parcelid' )
new_train_data_cabin.drop('Cabin',axis=1,inplace=True)
 new_train_data_cabin<drop_column>
def convert_label(dataframe):
lbI = LabelEncoder()
for c in dataframe.columns:
dataframe[c] = dataframe[c].fillna(0)
 if dataframe[c].dtype == 'object':
lbI.fit(list(dataframe[c].values))
dataframe[c] = lbI.transform(list(dataframe[c].values))
convert_label(df_train)
 convert_label(df_test)
new_test_data.drop('Cabin',axis=1,inplace=True)
 new_test_data<feature_engineering>
x_train = df_train.drop(['parcelid', 'logerror', 'transactiondate', 'propertyzoningdesc',
'propertycountylandusecode', ], axis=1)
 x_test = df_test.drop(['parcelid', 'propertyzoningdesc',
'propertycountylandusecode', '201610', '201611',
'201612', '201710', '201711', '201712'], axis = 1)
 x_train = x_train.values
y_train = df_train['logerror'].values
<feature_engineering><EOS> 
from datetime import datetime