Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_pretrained>
import pandas as pd
import numpy as np
import seaborn as sns
from matplotlib import pyplot as plt
def save_models(models):
for i, model in enumerate(models):
model.save_model('checkpoints/lgb_' + str(i))
print("Saved {} LightGBM models to files.".format(len(models)))
 def load_models(paths):
models = []
for path in paths:
model = lgb.Booster(model_file=path)
 models.append(model)
 return models<drop_column>
df_train = pd.read_csv('.. /input/train.csv' ).drop('Id',axis=1)
 df_train = df_train[df_train["GrLivArea"]<4500]
def lgb_drop_features(features):
unused_feature_list = ['parcelid', 'logerror']
missing_list = ['framing_id', 'architecture_style_id', 'story_id', 'perimeter_area', 'basement_sqft', 'storage_sqft']
unused_feature_list += missing_list
bad_feature_list = ['fireplace_flag', 'deck_id', 'pool_unk_1', 'construction_id', 'county_id', 'fips']
unused_feature_list += bad_feature_list
unused_feature_list += ['county_landuse_code_id', 'zoning_description_id']
return features.drop(unused_feature_list, axis=1, errors='ignore' )<create_dataframe>
sum(df_train.isna().sum() !=0 )
%%time
features_2016 = prop_2016
features_2017 = prop_2017
train = train<drop_column>
print('Categorical variables:',sum(df_train.dtypes=='O'))
print('Categorical variables:',sum(df_train.dtypes!='O'))
lgb_features = lgb_drop_features(train)
 print("Number of features for LightGBM: {}".format(len(lgb_features.columns)))
 lgb_features.head(5)
<train_model>
def fixskew(data):
return boxcox1p(data,boxcox_normmax(data+1))
lgb_label = train.logerror.astype(np.float32)
 print(lgb_label.head())
 lgb_X = lgb_features.values
lgb_y = lgb_label.values
np.random.seed(42)
 random.seed(10)
 X_train, X_val, y_train, y_val = train_test_split(lgb_X, lgb_y, test_size=0.2)
 outlier_threshold = 0.4
mask =(abs(y_train)<= outlier_threshold)
 X_train = X_train[mask, :]
y_train = y_train[mask]
print("X_train shape: {}".format(X_train.shape))
print("y_train shape: {}".format(y_train.shape))
print("X_val shape: {}".format(X_val.shape))
print("y_val shape: {}".format(y_val.shape))<define_variables>
y = np.log(df_train['SalePrice'])
 df_test = pd.read_csv('.. /input/test.csv' ).drop('Id',axis=1)
 features = df_train.drop('SalePrice',axis=1 ).append(df_test)
 features = features.drop(['Utilities', 'Street', 'PoolQC'], axis=1)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF1']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['TotalSF2'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['TotalBath'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['TotalPorchSF'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
 features['HasPool'] =(features['PoolArea']!=0 ).astype('int')
 features['Has2ndFloor'] =(features['2ndFlrSF']!=0 ).astype('int')
 features['HasGarage'] =(features['GarageArea']!=0 ).astype('int')
 features['HasBsmt'] =(features['TotalBsmtSF']!=0 ).astype('int')
 features['HasFireplace'] =(features['Fireplaces']!=0 ).astype('int')
 numcols = features.columns[features.dtypes!='O']
for col in numcols:
features[col]=features[col].fillna(features[col].mean())
 catcols = features.columns[features.dtypes=='O']
for col in catcols:
target_map = df_train['SalePrice'].groupby(df_train[col].fillna('None')).mean()
features[col+'_target']=features[col].map(target_map ).fillna(df_train['SalePrice'].mean())
 features = pd.get_dummies(features.fillna('None'))
X = features[:len(y)]
X_test = features[len(y):]
outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers])
 overfit = []
for col in X.columns:
counts = X[col].value_counts()
num_zeros = counts.iloc[0]
if num_zeros / len(X)* 100.0 > 99.0:
overfit.append(col)
 X = X.drop(overfit, axis=1)
 X_test = X_test.drop(overfit, axis=1 )
feature_names = [s for s in lgb_features.columns]
categorical_features = ['cooling_id', 'heating_id', 'landuse_type_id', 'year', 'month', 'quarter']
categorical_indices = []
for i, n in enumerate(lgb_features.columns):
if n in categorical_features:
categorical_indices.append(i)
 print(categorical_indices )<init_hyperparams>
kf = KFold(n_splits=5,shuffle=True)
 def scoreit(model,X=X):
return(np.sqrt(-cross_val_score(model,X,y,cv=kf,scoring='neg_mean_squared_error')) )
params = {}
params['objective'] = 'regression'
params['metric'] = 'mae'
params['num_threads'] = 4
params['boosting_type'] = 'gbdt'
params['num_boost_round'] = 2000
params['learning_rate'] = 0.003
params['early_stopping_rounds'] = 30
params['num_leaves'] = 127
params['min_data'] = 150
params['min_hessian'] = 0.001
params['max_depth'] = -1
params['max_bin'] = 255
params['sub_feature'] = 0.5
params['bagging_fraction'] = 0.7
params['bagging_freq'] = 50
params['min_data_per_group'] = 100
params['cat_smooth'] = 15.0
params['lambda_l1'] = 0.0
params['lambda_l2'] = 0.0
params['feature_fraction_seed'] = 2
params['bagging_seed'] = 3<train_model>
lasso = make_pipeline(RobustScaler() , Lasso(alpha=3e-4))
scores = scoreit(lasso,X)
 print("RMSE:",scores.mean() ,"+/-",scores.std() )
lgb_train_set = lgb.Dataset(X_train, label=y_train, feature_name=feature_names)
 lgb_valid_set = lgb.Dataset(X_val, label=y_val, feature_name=feature_names)
 np.random.seed(42)
 random.seed(36)
 model = lgb.train(params, lgb_train_set, verbose_eval=False,
valid_sets=[lgb_train_set, lgb_valid_set], valid_names=['train', 'val'],
categorical_feature=categorical_indices)
 print("Train score: {}".format(abs(model.predict(X_train)- y_train ).mean() * 100))
print("Val score: {}".format(abs(model.predict(X_val)- y_val ).mean() * 100))<train_model>
randomforest = RandomForestRegressor(n_estimators=100)
 scores = scoreit(randomforest,X)
 print("RMSE:",scores.mean() ,"+/-",scores.std() )
del params['early_stopping_rounds']
params['num_boost_round'] = 1250
print(params)
 outlier_threshold = 0.4
mask =(abs(lgb_y)<= outlier_threshold)
 lgb_X = lgb_X[mask, :]
lgb_y = lgb_y[mask]
lgb_train_set = lgb.Dataset(lgb_X, label=lgb_y, feature_name=feature_names)
 print("lgb_X: {}".format(lgb_X.shape))
print("lgb_y: {}".format(lgb_y.shape))
np.random.seed(42)
 random.seed(36)
 model = lgb.train(params, lgb_train_set, verbose_eval=True, categorical_feature=categorical_indices)
 print("score: {}".format(abs(model.predict(X_val)- y_val ).mean() * 100))<feature_engineering>
xgb = make_pipeline(RobustScaler() , XGBRegressor(
 learning_rate=0.01, n_estimators=3400, max_depth=3, min_child_weight=0,
gamma=0, subsample=0.75, colsample_bytree=0.7, objective='reg:linear', nthread=-1,
scale_pos_weight=1, reg_alpha=0.00005))
scores = scoreit(xgb,X)
 print("RMSE:",scores.mean() ,"+/-",scores.std() )
def transform_test_features(features_2016, features_2017):
test_features_2016 = lgb_drop_features(features_2016)
 test_features_2017 = lgb_drop_features(features_2017)
 test_features_2016['year'] = 0
test_features_2017['year'] = 1
test_features_2016['month'] = 10
test_features_2017['month'] = 10
test_features_2016['quarter'] = 4
test_features_2017['quarter'] = 4
return test_features_2016, test_features_2017

 def predict_and_export(models, features_2016, features_2017, file_name):
submission_2016 = pd.DataFrame()
submission_2017 = pd.DataFrame()
submission_2016['ParcelId'] = features_2016.parcelid
submission_2017['ParcelId'] = features_2017.parcelid
test_features_2016, test_features_2017 = transform_test_features(features_2016, features_2017)
 pred_2016, pred_2017 = [], []
for i, model in enumerate(models):
print("Start model {}(2016)".format(i))
pred_2016.append(model.predict(test_features_2016))
print("Start model {}(2017)".format(i))
pred_2017.append(model.predict(test_features_2017))
mean_pred_2016 = np.mean(pred_2016, axis=0)
 mean_pred_2017 = np.mean(pred_2017, axis=0)
 submission_2016['201610'] = [float(format(x, '.4f')) for x in mean_pred_2016]
submission_2016['201611'] = submission_2016['201610']
submission_2016['201612'] = submission_2016['201610']
submission_2017['201710'] = [float(format(x, '.4f')) for x in mean_pred_2017]
submission_2017['201711'] = submission_2017['201710']
submission_2017['201712'] = submission_2017['201710']
submission = submission_2016.merge(how='inner', right=submission_2017, on='ParcelId')
 print("Length of submission DataFrame: {}".format(len(submission)))
 print("Submission header:")
 print(submission.head())
 submission.to_csv(file_name, index=False)
 return submission, pred_2016, pred_2017<predict_on_test>
stack = StackingRegressor([lasso,xgb,randomforest],meta_regressor=xgb)
 scores = scoreit(stack,X)
 print("RMSE:",scores.mean() ,"+/-",scores.std() )
<install_modules><EOS> 
lasso = make_pipeline(RobustScaler() , Lasso(alpha=3e-4))
lasso.fit(X,y)
 pred = pd.Series(np.exp(lasso.predict(X_test)))
 test_id = pd.read_csv('.. /input/test.csv')['Id']
output = pd.DataFrame({'Id':test_id, 'SalePrice':pred})
 output.to_csv('submission.csv', index=False )