Kaggle competition: House Prices - Advanced Regression Techniques
from tqdm import tqdm<import_modules>
params = {'legend.fontsize': 'x-large',
'figure.figsize':(10,8),
'axes.labelsize': 'x-large',
'axes.labelcolor': '
'axes.titlesize':'15',
'text.color':'green',
'axes.titlepad': 35,
'xtick.labelsize':'small',
'ytick.labelsize':'small'}
plt.rcParams.update(params)
 warnings.filterwarnings('ignore' )
import gzip<define_search_space>
train_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 train_df.head()
ff = 1.0
weights = [ 0.292, 0.165, 0.147, 0.200, 0.196 ] 
<count_values>
test_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 test_df.head()
print(sum(weights))<feature_engineering>
saleprice_train = train_df.SalePrice
if CORRECT_FOR_BUG:
weights[1] /= 10<define_variables>
housing_df = pd.concat([train_df.drop('SalePrice',axis=1),test_df] )
fnames =('output_dataset2017a.csv',
'finalxgb20171015_145310.csv',
'2017Q4.csv',
'catboost_opt4.csv',
'simplad2017.csv')
 output="withSimpLADNoFudge.csv"<save_to_csv>
housing_df['Id'].nunique()
if ACTUALLY_DO_SOMETHING:
for id, r in zip(ids,result):
fs.write(id)
 for j in range(6):
fs.write("," +str(round(r[j],6)))
 fs.write("
" )<set_options>
mis_val_percent = round(( 100 * housing_df.isnull().sum() / len(housing_df)) ,2 )
if ACTUALLY_DO_SOMETHING:
fs.close()<load_pretrained>
mis_cols=mis_val_percent.loc[(mis_val_percent>0)].sort_values(ascending=False)
 mis_cols
if ACTUALLY_DO_SOMETHING:
f_in = open(output, 'rb')
 f_out = gzip.open(output+'.gz', 'wb')
 f_out.writelines(f_in)
 f_out.close()
f_in.close()<train_model>
housing_df.drop(mis_cols[mis_cols>45].index,inplace=True,axis=1)
 housing_df.info()
2+2<import_modules>
cols_to_drop = mis_cols[mis_cols>45].index
cols_to_drop
import numpy as np
import pandas as pd
import xgboost as xgb
from sklearn.preprocessing import LabelEncoder
import lightgbm as lgb
import gc
from sklearn.linear_model import LinearRegression
import random
import datetime as dt
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout, BatchNormalization
from keras.layers.advanced_activations import PReLU
from keras.layers.noise import GaussianDropout
from keras.optimizers import Adam
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import Imputer 
<define_variables>
housing_df.drop('MiscVal',axis=1,inplace=True )
FUDGE_FACTOR = 1.1200
XGB_WEIGHT = 0.6200
BASELINE_WEIGHT = 0.0100
OLS_WEIGHT = 0.0620
NN_WEIGHT = 0.0800
XGB1_WEIGHT = 0.8000
BASELINE_PRED = 0.0115<load_from_csv>
mis_cols[mis_cols<45]
print("
Reading data from disk...")
 prop = pd.read_csv('.. /input/properties_2016.csv')
 train = pd.read_csv(".. /input/train_2016_v2.csv" )<data_type_conversions>
Garage_missing = mis_cols[categorical_col[0:4]].index
Garage_missing
print("
Processing data for LightGBM...")
 for c, dtype in zip(prop.columns, prop.dtypes):	
if dtype == np.float64:
prop[c] = prop[c].astype(np.float32)
 df_train = train.merge(prop, how='left', on='parcelid')
 missing_perc_thresh = 0.98
num_rows = df_train.shape[0]
for c in df_train.columns:
num_missing = df_train[c].isnull().sum()
if num_missing == 0:
continue
missing_frac = num_missing / float(num_rows)
 if missing_frac > missing_perc_thresh:
df_train.drop([c],axis=1)
 for c in df_train.columns:
num_uniques = len(df_train[c].unique())
 if df_train[c].isnull().sum() != 0:
num_uniques -= 1
if num_uniques == 1:
df_train.drop([c],axis=1)
 df_train.fillna(df_train.median() ,inplace = True)
 x_train = df_train.drop(['parcelid', 'logerror', 'transactiondate', 'propertyzoningdesc','propertycountylandusecode', 'fireplacecnt', 'fireplaceflag'], axis=1)
 y_train = df_train['logerror'].values
print(x_train.shape, y_train.shape)
 train_columns = x_train.columns
for c in x_train.dtypes[x_train.dtypes == object].index.values:
x_train[c] =(x_train[c] == True)
 del df_train; gc.collect()
x_train = x_train.values.astype(np.float32, copy=False)
 d_train = lgb.Dataset(x_train, label=y_train)
<init_hyperparams>
housing_df[Garage_missing]=housing_df[Garage_missing].replace(np.nan,'No Garage' )
params = {}
params['max_bin'] = 10
params['learning_rate'] = 0.0021
params['boosting_type'] = 'gbdt'
params['objective'] = 'regression'
params['metric'] = 'l1'
params['sub_feature'] = 0.345
params['bagging_fraction'] = 0.85
params['bagging_freq'] = 40
params['num_leaves'] = 512
params['min_data'] = 500
params['min_hessian'] = 0.05
params['verbose'] = 0
params['feature_fraction_seed'] = 2
params['bagging_seed'] = 3
np.random.seed(0)
 random.seed(0)
<load_from_csv>
Bsmt_missing = mis_cols[categorical_col[4:9]].index
Bsmt_missing
print("
Fitting LightGBM model...")
 clf = lgb.train(params, d_train, 430)
 del d_train; gc.collect()
del x_train; gc.collect()
print("
Prepare for LightGBM prediction...")
 print(" Read sample file...")
 sample = pd.read_csv('.. /input/sample_submission.csv')
 print("...")
 sample['parcelid'] = sample['ParcelId']
print(" Merge with property data...")
 df_test = sample.merge(prop, on='parcelid', how='left')
 print("...")
 del sample, prop; gc.collect()
print("...")
 x_test = df_test[train_columns]
print("...")
 del df_test; gc.collect()
print(" Preparing x_test...")
 for c in x_test.dtypes[x_test.dtypes == object].index.values:
x_test[c] =(x_test[c] == True)
 print("...")
 x_test = x_test.values.astype(np.float32, copy=False)
 print("
Start LightGBM prediction...")
 p_test = clf.predict(x_test)
 del x_test; gc.collect()
print("
Unadjusted LightGBM predictions:")
 print(pd.DataFrame(p_test ).head())
<load_from_csv>
housing_df[Bsmt_missing]=housing_df[Bsmt_missing].replace(np.nan,'No Basement' )
print("
Re-reading properties file...")
 properties = pd.read_csv('.. /input/properties_2016.csv')
<categorify>
housing_df[categorical_col[9]]=housing_df[categorical_col[9]].replace(np.nan,'None' )
print("
Processing data for XGBoost...")
 for c in properties.columns:
properties[c]=properties[c].fillna(-1)
 if properties[c].dtype == 'object':
lbl = LabelEncoder()
lbl.fit(list(properties[c].values))
properties[c] = lbl.transform(list(properties[c].values))
train_df = train.merge(properties, how='left', on='parcelid')
 missing_perc_thresh = 0.98
num_rows = train_df.shape[0]
for c in train_df.columns:
num_missing = train_df[c].isnull().sum()
if num_missing == 0:
continue
missing_frac = num_missing / float(num_rows)
 if missing_frac > missing_perc_thresh:
train_df.drop([c],axis=1)
 for c in train_df.columns:
num_uniques = len(train_df[c].unique())
 if train_df[c].isnull().sum() != 0:
num_uniques -= 1
if num_uniques == 1:
train_df.drop([c],axis=1)
 x_train = train_df.drop(['parcelid', 'logerror','transactiondate'], axis=1)
 x_test = properties.drop(['parcelid'], axis=1)
 print('Shape train: {}
Shape test: {}'.format(x_train.shape, x_test.shape))<train_model>
housing_df[categorical_col[10]]=housing_df[categorical_col[10]].replace(np.nan,'RL' )
print("
Setting up data for XGBoost...")
 xgb_params = {
'eta': 0.037,
'max_depth': 5,
'subsample': 0.80,
'objective': 'reg:linear',
'eval_metric': 'mae',
'lambda': 0.8,
'alpha': 0.4,
'base_score': y_mean,
'silent': 1
}
dtrain = xgb.DMatrix(x_train, y_train)
 dtest = xgb.DMatrix(x_test)
 num_boost_rounds = 250
print("num_boost_rounds="+str(num_boost_rounds))
print("
Training XGBoost...")
 model = xgb.train(dict(xgb_params, silent=1), dtrain, num_boost_round=num_boost_rounds)
 print("
Predicting with XGBoost...")
 xgb_pred1 = model.predict(dtest)
 print("
First XGBoost predictions:")
 print(pd.DataFrame(xgb_pred1 ).head())
<train_model>
housing_df[categorical_col[11]]=housing_df[categorical_col[11]].replace(np.nan,'AllPub' )
print("
Setting up data for XGBoost...")
 xgb_params = {
'eta': 0.033,
'max_depth': 6,
'subsample': 0.80,
'objective': 'reg:linear',
'eval_metric': 'mae',
'base_score': y_mean,
'silent': 1
}
num_boost_rounds = 150
print("num_boost_rounds="+str(num_boost_rounds))
print("
Training XGBoost again...")
 model = xgb.train(dict(xgb_params, silent=1), dtrain, num_boost_round=num_boost_rounds)
 print("
Predicting with XGBoost again...")
 xgb_pred2 = model.predict(dtest)
 print("
Second XGBoost predictions:")
 print(pd.DataFrame(xgb_pred2 ).head())
<prepare_output>
housing_df[categorical_col[12]]=housing_df[categorical_col[12]].replace(np.nan,'Typ' )
xgb_pred = XGB1_WEIGHT*xgb_pred1 +(1-XGB1_WEIGHT)*xgb_pred2
print("
Combined XGBoost predictions:")
 print(pd.DataFrame(xgb_pred ).head() )<drop_column>
housing_df[categorical_col[13]]=housing_df[categorical_col[13]].replace(np.nan,'VinylSd')
 housing_df[categorical_col[14]]=housing_df[categorical_col[14]].replace(np.nan,'VinylSd' )
del train_df
del x_train
del x_test
del properties
del dtest
del dtrain
del xgb_pred1
del xgb_pred2
gc.collect()<load_from_csv>
housing_df[categorical_col[15]]=housing_df[categorical_col[15]].replace(np.nan,'WD' )
np.random.seed(17)
 random.seed(17)
 print("
Processing data for OLS...")
 train = pd.read_csv(".. /input/train_2016_v2.csv", parse_dates=["transactiondate"])
 properties = pd.read_csv(".. /input/properties_2016.csv")
 submission = pd.read_csv(".. /input/sample_submission.csv")
 print(len(train),len(properties),len(submission))<feature_engineering>
housing_df[categorical_col[16]]=housing_df[categorical_col[16]].replace(np.nan,'SBrkr' )
def get_features(df):
df["transactiondate"] = pd.to_datetime(df["transactiondate"])
 df["transactiondate_year"] = df["transactiondate"].dt.year
df["transactiondate_month"] = df["transactiondate"].dt.month
df['transactiondate'] = df['transactiondate'].dt.quarter
df = df.fillna(-1.0)
 return df
def MAE(y, ypred):
return np.sum([abs(y[i]-ypred[i])for i in range(len(y)) ])/ len(y )<merge>
housing_df[categorical_col[17]]=housing_df[categorical_col[17]].replace(np.nan,'TA' )
train = pd.merge(train, properties, how='left', on='parcelid')
 y = train['logerror'].values
test = pd.merge(submission, properties, how='left', left_on='ParcelId', right_on='parcelid')
 properties = []
exc = [train.columns[c] for c in range(len(train.columns)) if train.dtypes[c] == 'O'] + ['logerror','parcelid']
col = [c for c in train.columns if c not in exc]
train = get_features(train[col])
 test['transactiondate'] = '2016-01-01'
test = get_features(test[col] )<train_model>
print("LotFrontage contains %s missing values"%(mis_cols.LotFrontage))
print("
Fitting OLS...")
 reg = LinearRegression(n_jobs=-1)
 reg.fit(train, y); print('fit...')
 print(MAE(y, reg.predict(train)))
 train = []; y = []
test_dates = ['2016-10-01','2016-11-01','2016-12-01','2017-10-01','2017-11-01','2017-12-01']
test_columns = ['201610','201611','201612','201710','201711','201712'] 
<define_variables>
train_df['LotFrontage']=train_df.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
test_df['LotFrontage']=test_df.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
print("
Combining XGBoost, LightGBM, NN, and baseline predicitons...")
 lgb_weight = 1 - XGB_WEIGHT - BASELINE_WEIGHT - OLS_WEIGHT
lgb_weight0 = lgb_weight /(1 - OLS_WEIGHT)
 xgb_weight0 = XGB_WEIGHT /(1 - OLS_WEIGHT)
 baseline_weight0 = BASELINE_WEIGHT /(1 - OLS_WEIGHT)
 pred0 = 0
pred0 += xgb_weight0*xgb_pred
pred0 += baseline_weight0*BASELINE_PRED
pred0 += lgb_weight0*p_test 
<predict_on_test>
Garage_cols = int_mis_cols[int_mis_cols.str.contains('Gar')]
Basement_cols = int_mis_cols[int_mis_cols.str.contains('Bsmt')]
print(Garage_cols,"
",Basement_cols )
print("
Combined XGB/LGB/NN/baseline predictions:")
 print(pd.DataFrame(pred0 ).head())
 print("
Predicting with OLS and combining with XGB/LGB/NN/baseline predicitons:...")
 for i in range(len(test_dates)) :
test['transactiondate'] = test_dates[i]
pred = FUDGE_FACTOR *(OLS_WEIGHT*reg.predict(get_features(test)) +(1-OLS_WEIGHT)*pred0)
 submission[test_columns[i]] = [float(format(x, '.4f')) for x in pred]
print('predict...', i)
 print("
Combined XGB/LGB/NN/baseline/OLS predictions:")
 print(submission.head() )<save_to_csv>
housing_df[Garage_cols]=housing_df[Garage_cols].replace(np.nan,0)
 housing_df[Basement_cols]=housing_df[Basement_cols].replace(np.nan,0)
 housing_df['MasVnrArea']=housing_df['MasVnrArea'].replace(np.nan,0 )
print("
Writing results to disk...")
 submission.to_csv('sub{}.csv'.format(datetime.now().strftime('%Y%m%d_%H%M%S')) , index=False)
 print("
Finished..." )<define_variables>
mis_val_percent = round(( 100 * housing_df.isnull().sum() / len(housing_df)) ,2)
 mis_val_percent[mis_val_percent>0]
FUDGE_FACTOR = 1.1200
XGB_WEIGHT = 0.6200
BASELINE_WEIGHT = 0.0100
OLS_WEIGHT = 0.0620
NN_WEIGHT = 0.0800
XGB1_WEIGHT = 0.8000
BASELINE_PRED = 0.0115<import_modules>
housing_df.columns.nunique()
import numpy as np
import pandas as pd
import xgboost as xgb
import random
import lightgbm as lgb
import datetime as dt
import matplotlib.pyplot as plt
import seaborn as sns
import gc
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score<import_modules>
train_df.SalePrice=np.log(train_df.SalePrice )
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout, BatchNormalization
from keras.layers.advanced_activations import PReLU
from keras.layers.noise import GaussianDropout
from keras.optimizers import Adam
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import Imputer<load_from_csv>
curr_year=datetime.datetime.now().year
housing_df['YearBuilt'] = curr_year - housing_df['YearBuilt']
housing_df['YearRemodAdd'] = curr_year - housing_df['YearRemodAdd']
housing_df['GarageYrBlt'] = curr_year - housing_df['GarageYrBlt']
housing_df['YrSold'] = curr_year - housing_df['YrSold']
train_2017 = pd.read_csv('.. /input/train_2017.csv',parse_dates = ["transactiondate"])
 train_2016 = pd.read_csv('.. /input/train_2016_v2.csv',parse_dates = ["transactiondate"])
 frames = [train_2016,train_2017]
train = pd.concat(frames)
 properties = pd.read_csv('.. /input/properties_2017.csv')
 test = pd.read_csv('.. /input/sample_submission.csv')
 test = test.rename(columns = {'ParcelId': 'parcelid'} )<train_model>
Numerics=['int64','float64']
integer_cols=housing_df.select_dtypes(include=Numerics)
 integer_cols.drop('Id',axis=1,inplace=True)
 int_cols = integer_cols.nunique()
int_cols[int_cols<50]
print("Training Size:" + str(train.shape))
print("Property Size:" + str(properties.shape))
print("Sample Size:" + str(test.shape)) 
<prepare_x_and_y>
housing_df['MSSubClass'] = housing_df['MSSubClass'].astype('object')
 housing_df['OverallQual'] = housing_df['OverallQual'].astype('object')
 housing_df['OverallCond'] = housing_df['OverallCond'].astype('object')
 housing_df['BsmtFullBath'] = housing_df['BsmtFullBath'].astype('object')
 housing_df['BsmtHalfBath'] = housing_df['BsmtHalfBath'].astype('object')
 housing_df['FullBath'] = housing_df['FullBath'].astype('object')
 housing_df['HalfBath'] = housing_df['HalfBath'].astype('object')
 housing_df['BedroomAbvGr'] = housing_df['BedroomAbvGr'].astype('object')
 housing_df['KitchenAbvGr'] = housing_df['KitchenAbvGr'].astype('object')
 housing_df['TotRmsAbvGrd'] = housing_df['TotRmsAbvGrd'].astype('object')
 housing_df['Fireplaces'] = housing_df['Fireplaces'].astype('object')
 housing_df['GarageCars'] = housing_df['GarageCars'].astype('object' )
print("
Processing data for LightGBM...")
 for c, dtype in zip(properties.columns, properties.dtypes):
if dtype == np.float64:
properties[c] = properties[c].astype(np.float32)
 df_train = train.merge(properties, how='left', on='parcelid')
 df_train.fillna(df_train.median() ,inplace = True)
 x_train = df_train.drop(['parcelid', 'logerror', 'transactiondate', 'propertyzoningdesc',
'propertycountylandusecode', 'fireplacecnt', 'fireplaceflag'], axis=1)
 y_train = df_train['logerror'].values
print(x_train.shape, y_train.shape )<data_type_conversions>
col_cat=housing_df.nunique()
binary_cols=col_cat[col_cat<3]
binary_cols.index
train_columns = x_train.columns
for c in x_train.dtypes[x_train.dtypes == object].index.values:
x_train[c] =(x_train[c] == True)
 del df_train; gc.collect()
x_train = x_train.values.astype(np.float32, copy=False)
 d_train = lgb.Dataset(x_train, label=y_train )<init_hyperparams>
housing_df.drop(['LowQualFinSF','3SsnPorch','PoolArea'],axis=1,inplace=True )
params = {}
params['max_bin'] = 10
params['learning_rate'] = 0.0021
params['boosting_type'] = 'gbdt'
params['objective'] = 'regression'
params['metric'] = 'l1'
params['sub_feature'] = 0.345
params['bagging_fraction'] = 0.85
params['bagging_freq'] = 40
params['num_leaves'] = 512
params['min_data'] = 500
params['min_hessian'] = 0.05
params['verbose'] = 0
params['feature_fraction_seed'] = 2
params['bagging_seed'] = 3
np.random.seed(0)
 random.seed(0 )<train_model>
housing_df['Street']=housing_df['Street'].map({'Pave': 1, 'Grvl': 0})
 housing_df['Utilities']=housing_df['Utilities'].map({'AllPub': 1,'NoSeWa':0})
 housing_df['CentralAir']=housing_df['CentralAir'].map({'Y': 1, "N": 0} )
print("
Fitting LightGBM model...")
 clf = lgb.train(params, d_train, 430 )<set_options>
categorical_fields=housing_df.select_dtypes(include='object')
 categorical_columns=categorical_fields.nunique().sort_values(ascending=False ).index
categorical_columns
del d_train; gc.collect()
del x_train; gc.collect()<feature_engineering>
dummy_vars = pd.get_dummies(housing_df[categorical_columns], drop_first=True)
 dummy_vars.head()
print("
Prepare for LightGBM prediction...")
 print(" Read sample file...")
 sample = pd.read_csv('.. /input/sample_submission.csv')
 print("...")
 sample['parcelid'] = sample['ParcelId']
print(" Merge with property data...")
 df_test = sample.merge(properties, on='parcelid', how='left')
 print("...")
 del sample, properties; gc.collect()
print("...")
 x_test = df_test[train_columns]
print("...")
 del df_test; gc.collect()
print(" Preparing x_test...")
 for c in x_test.dtypes[x_test.dtypes == object].index.values:
x_test[c] =(x_test[c] == True)
 print("...")
 x_test = x_test.values.astype(np.float32, copy=False)
 print("
Start LightGBM prediction...")
 p_test = clf.predict(x_test)
 del x_test; gc.collect()
print("
Unadjusted LightGBM predictions:")
 print(pd.DataFrame(p_test ).head())
<load_from_csv>
housing_df = pd.concat([housing_df, dummy_vars], axis=1)
 housing_df = housing_df.drop(categorical_columns, axis = 1)
 housing_df.shape
print("
Re-reading properties file...")
 properties = pd.read_csv('.. /input/properties_2017.csv' )<categorify>
df_train = housing_df.iloc[:1460]
df_test = housing_df.iloc[1460:]
df_train['LotFrontage']=train_df['LotFrontage']
df_train['SalePrice'] = train_df['SalePrice']
df_test['LotFrontage']=test_df['LotFrontage']
df_test['SalePrice'] = 0
print("
Processing data for XGBoost...")
 for c in properties.columns:
properties[c]=properties[c].fillna(-1)
 if properties[c].dtype == 'object':
lbl = LabelEncoder()
lbl.fit(list(properties[c].values))
properties[c] = lbl.transform(list(properties[c].values))
train_df = train.merge(properties, how='left', on='parcelid')
 x_train = train_df.drop(['parcelid', 'logerror','transactiondate'], axis=1)
 x_test = properties.drop(['parcelid'], axis=1 )<prepare_x_and_y>
df_train.drop('Id',axis=1,inplace=True )
train_df=train_df[ train_df.logerror > -0.4 ]
train_df=train_df[ train_df.logerror < 0.419 ]
x_train=train_df.drop(['parcelid', 'logerror','transactiondate'], axis=1)
 y_train = train_df["logerror"].values.astype(np.float32)
 y_mean = np.mean(y_train)
 print('After removing outliers:')
 print('Shape train: {}
Shape test: {}'.format(x_train.shape, x_test.shape))<init_hyperparams>
y_train = df_train.pop('SalePrice')
 X_train = df_train
scaler = MinMaxScaler()
X_train_scaled = scaler.fit_transform(X_train.values)
 X_train = pd.DataFrame(X_train_scaled, index=X_train.index, columns=X_train.columns)
 X_train.head()
print("
Setting up data for XGBoost...")
 xgb_params = {
'eta': 0.037,
'max_depth': 5,
'subsample': 0.80,
'objective': 'reg:linear',
'eval_metric': 'mae',
'lambda': 0.8,
'alpha': 0.4,
'base_score': y_mean,
'silent': 1
}<train_model>
params = {'alpha': [0.0001, 0.001, 0.01, 0.05, 0.1,
0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 2.0, 3.0,
4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 20, 50, 100]}
ridge = Ridge()
folds = 5
model_cv = GridSearchCV(estimator = ridge,
param_grid = params,
scoring= 'neg_mean_absolute_error',
cv = folds,
return_train_score=True,
verbose = 1)
 model_cv.fit(X_train, y_train)
 print(model_cv.best_params_)
 print(model_cv.best_score_ )
dtrain = xgb.DMatrix(x_train, y_train)
 dtest = xgb.DMatrix(x_test)
 num_boost_rounds = 250
print("num_boost_rounds="+str(num_boost_rounds))
print("
Training XGBoost...")
 model = xgb.train(dict(xgb_params, silent=1), dtrain, num_boost_round=num_boost_rounds)
 print("
Predicting with XGBoost...")
 xgb_pred1 = model.predict(dtest)
 print("
First XGBoost predictions:")
 print(pd.DataFrame(xgb_pred1 ).head() )<train_model>
alpha = 2
ridge = Ridge(alpha=alpha)
 ridge.fit(X_train, y_train)
 r_coeff=ridge.coef_
r_coeff[r_coeff!=0].shape
print("
Setting up data for XGBoost...")
 xgb_params = {
'eta': 0.033,
'max_depth': 6,
'subsample': 0.80,
'objective': 'reg:linear',
'eval_metric': 'mae',
'base_score': y_mean,
'silent': 1
}
num_boost_rounds = 150
print("num_boost_rounds="+str(num_boost_rounds))
print("
Training XGBoost again...")
 model = xgb.train(dict(xgb_params, silent=1), dtrain, num_boost_round=num_boost_rounds)
 print("
Predicting with XGBoost again...")
 xgb_pred2 = model.predict(dtest)
 print("
Second XGBoost predictions:")
 print(pd.DataFrame(xgb_pred2 ).head() )<drop_column>
y_train_pred = ridge.predict(X_train)
 RR2=metrics.r2_score(y_train, y_train_pred)
 print("Ridge R squared(train):",RR2 )
xgb_pred = XGB1_WEIGHT*xgb_pred1 +(1-XGB1_WEIGHT)*xgb_pred2
print("
Combined XGBoost predictions:")
 print(pd.DataFrame(xgb_pred ).head())
 del train_df
del x_train
del x_test
del properties
del dtest
del dtrain
del xgb_pred1
del xgb_pred2
gc.collect()<load_from_csv>
lasso = Lasso()
model_cv = GridSearchCV(estimator = lasso,
param_grid = params,
scoring= 'neg_mean_absolute_error',
cv = folds,
return_train_score=True,
verbose = 1)
 model_cv.fit(X_train, y_train )
print("
Processing data for Neural Network...")
 print('
Loading train, prop and sample data...')
 train_2017 = pd.read_csv('.. /input/train_2017.csv',parse_dates = ["transactiondate"])
 train_2016 = pd.read_csv('.. /input/train_2016_v2.csv',parse_dates = ["transactiondate"])
 frames = [train_2016,train_2017]
train = pd.concat(frames)
 prop = pd.read_csv('.. /input/properties_2017.csv')
 sample = pd.read_csv('.. /input/sample_submission.csv' )<categorify>
print(model_cv.best_params_)
 print(model_cv.best_score_ )
print('Fitting Label Encoder on properties...')
 for c in prop.columns:
prop[c]=prop[c].fillna(-1)
 if prop[c].dtype == 'object':
lbl = LabelEncoder()
lbl.fit(list(prop[c].values))
prop[c] = lbl.transform(list(prop[c].values))<feature_engineering>
alpha = 0.0001
lasso = Lasso(alpha=alpha)
 lasso.fit(X_train, y_train)
 lasso_c=lasso.coef_
lasso_c[lasso_c!=0].shape
print('Creating training set...')
 df_train = train.merge(prop, how='left', on='parcelid')
 df_train["transactiondate"] = pd.to_datetime(df_train["transactiondate"])
 df_train["transactiondate_year"] = df_train["transactiondate"].dt.year
df_train["transactiondate_month"] = df_train["transactiondate"].dt.month
df_train['transactiondate_quarter'] = df_train['transactiondate'].dt.quarter
df_train["transactiondate"] = df_train["transactiondate"].dt.day<train_model>
y_train_pred = lasso.predict(X_train)
 LR2=metrics.r2_score(y_true=y_train, y_pred=y_train_pred)
 print("Lasso R squared(Train)",LR2 )
print('Filling NA/NaN values...')
 df_train.fillna(-1.0 )<prepare_x_and_y>
df_test_final = df_test.copy()
df_test.pop('Id')
 df_test.head()
print('Creating x_train and y_train from df_train...')
 x_train = df_train.drop(['parcelid', 'logerror', 'transactiondate', 'propertyzoningdesc', 'propertycountylandusecode','fireplacecnt', 'fireplaceflag'], axis=1)
 y_train = df_train["logerror"]<feature_engineering>
y_test = df_test['SalePrice']
X_test = df_test.loc[:, df_test.columns != 'SalePrice']
X_test_scaled = scaler.transform(X_test.values)
 X_test = pd.DataFrame(X_test_scaled, index=X_test.index, columns=X_test.columns )
y_mean = np.mean(y_train)
 print(x_train.shape, y_train.shape)
 train_columns = x_train.columns
for c in x_train.dtypes[x_train.dtypes == object].index.values:
x_train[c] =(x_train[c] == True )<feature_engineering>
y_test_pred_lasso = np.exp(lasso.predict(X_test))
print('Creating df_test...')
 sample['parcelid'] = sample['ParcelId']
print("Merging Sample with property data...")
 df_test = sample.merge(prop, on='parcelid', how='left')
 df_test["transactiondate"] = pd.to_datetime('2016-11-15')
 df_test["transactiondate_year"] = df_test["transactiondate"].dt.year
df_test["transactiondate_month"] = df_test["transactiondate"].dt.month
df_test['transactiondate_quarter'] = df_test['transactiondate'].dt.quarter
df_test["transactiondate"] = df_test["transactiondate"].dt.day
x_test = df_test[train_columns]<normalization>
y_test_pred_ridge = np.exp(ridge.predict(X_test))
print("
Preprocessing neural network data...")
 imputer= Imputer()
imputer.fit(x_train.iloc[:, :])
 x_train = imputer.transform(x_train.iloc[:, :])
 imputer.fit(x_test.iloc[:, :])
 x_test = imputer.transform(x_test.iloc[:, :])
 sc = StandardScaler()
x_train = sc.fit_transform(x_train)
 x_test = sc.transform(x_test)
 len_x=int(x_train.shape[1])
 print("len_x is:",len_x )<choose_model_class>
y_test_final =(0.2*y_test_pred_lasso+0.8*y_test_pred_ridge )
print("
Setting up neural network model...")
 nn = Sequential()
nn.add(Dense(units = 400 , kernel_initializer = 'normal', input_dim = len_x))
nn.add(PReLU())
 nn.add(Dropout (.4))
nn.add(Dense(units = 160 , kernel_initializer = 'normal'))
nn.add(PReLU())
 nn.add(BatchNormalization())
 nn.add(Dropout (.6))
nn.add(Dense(units = 64 , kernel_initializer = 'normal'))
nn.add(PReLU())
 nn.add(BatchNormalization())
 nn.add(Dropout (.5))
nn.add(Dense(units = 26, kernel_initializer = 'normal'))
nn.add(PReLU())
 nn.add(BatchNormalization())
 nn.add(Dropout (.6))
nn.add(Dense(1, kernel_initializer='normal'))
nn.compile(loss='mae', optimizer=Adam(lr=4e-3, decay=1e-4))<train_model>
my_submission = pd.DataFrame({'Id': test_df.Id, 'SalePrice': y_test_final})
 my_submission.to_csv('submission.csv', index=False )