Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<categorify>
warnings.filterwarnings('ignore')
 main_file_path = '.. /input/train.csv'
data = pd.read_csv(main_file_path )
for variable in tqdm(frequency_encoded_variables):
freq_enc_dict = frequency_encoding(variable)
 train[variable] = train[variable].map(lambda x: freq_enc_dict.get(x, np.nan))
test[variable] = test[variable].map(lambda x: freq_enc_dict.get(x, np.nan))
categorical_columns.remove(variable )<feature_engineering>
test_data = pd.read_csv('.. /input/test.csv')
 X_test = test_data.copy(deep=True )
indexer = {}
for col in tqdm(categorical_columns):
if col == 'MachineIdentifier': continue
_, indexer[col] = pd.factorize(train[col])
 for col in tqdm(categorical_columns):
if col == 'MachineIdentifier': continue
train[col] = indexer[col].get_indexer(train[col])
 test[col] = indexer[col].get_indexer(test[col] )<drop_column>
data.drop(data[(data["GrLivArea"]>4000)&(data["SalePrice"]<300000)].index,inplace=True)
 X = data.drop('SalePrice', axis=1 ).drop('Id', axis=1)
 ID = data['Id']
y = data['SalePrice']
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )<prepare_x_and_y>
test_ID = test_data['Id']
test_data.drop('Id', axis=1, inplace=True )
target = train['HasDetections']
del train['HasDetections']<init_hyperparams>
train_size = X.shape[0]
all_data = pd.concat(( X, test_data)).reset_index(drop=True )
param = {'num_leaves': 60,
'min_data_in_leaf': 60,
'objective':'binary',
'max_depth': -1,
'learning_rate': 0.1,
"boosting": "gbdt",
"feature_fraction": 0.8,
"bagging_freq": 1,
"bagging_fraction": 0.8 ,
"bagging_seed": 11,
"metric": 'auc',
"lambda_l1": 0.1,
"random_state": 133,
"verbosity": -1}<define_variables>
ordinals = ['YearBuilt', 'YearRemodAdd', 'GarageYrBlt', 'MoSold', 'YrSold']
for ordinal in ordinals:
all_data[ordinal] = all_data[ordinal].astype(str )
max_iter = 5<set_options>
qual_dict = {None: 0, "Po": 1, "Fa": 2, "TA": 3, "Gd": 4, "Ex": 5, np.NaN:0}
all_data["ExterQual"] = all_data["ExterQual"].apply(lambda x: qual_dict[x])
 all_data["ExterCond"] = all_data["ExterCond"].apply(lambda x: qual_dict[x])
 all_data["BsmtQual"] = all_data["BsmtQual"].apply(lambda x: qual_dict[x])
 all_data["BsmtCond"] = all_data["BsmtCond"].apply(lambda x: qual_dict[x])
 all_data["HeatingQC"] = all_data["HeatingQC"].apply(lambda x: qual_dict[x])
 all_data["KitchenQual"] = all_data["KitchenQual"].apply(lambda x: qual_dict[x])
 all_data["FireplaceQu"] = all_data["FireplaceQu"].apply(lambda x: qual_dict[x])
 all_data["GarageQual"] = all_data["GarageQual"].apply(lambda x: qual_dict[x])
 all_data["GarageCond"] = all_data["GarageCond"].apply(lambda x: qual_dict[x])
 all_data["BsmtExposure"] = all_data["BsmtExposure"].apply(lambda x:
{None: 0, "No": 1, "Mn": 2, "Av": 3, "Gd": 4, np.NaN: 0}[x])
 bsmt_fin_dict = {None: 0, "Unf": 1, "LwQ": 2, "Rec": 3, "BLQ": 4, "ALQ": 5, "GLQ": 6, np.NaN:0}
all_data["BsmtFinType1"] = all_data["BsmtFinType1"].apply(lambda x: bsmt_fin_dict[x])
 all_data["BsmtFinType2"] = all_data["BsmtFinType2"].apply(lambda x: bsmt_fin_dict[x])
 all_data["Functional"] = all_data["Functional"].apply(lambda x:
{None: 0, "Sal": 1, "Sev": 2, "Maj2": 3, "Maj1": 4,
"Mod": 5, "Min2": 6, "Min1": 7, "Typ": 8, np.NaN: 0}[x])
 all_data["GarageFinish"] = all_data["GarageFinish"].apply(lambda x:
{None: 0, "Unf": 1, "RFn": 2, "Fin": 3, np.NaN: 0}[x])
 all_data["Fence"] = all_data["Fence"].apply(
 lambda x: {None: 0, "MnWw": 1, "GdWo": 2, "MnPrv": 3, "GdPrv": 4, np.NaN: 0}[x])
 all_data["YearBuilt"] = all_data["YearBuilt"]
all_data["YearRemodAdd"] = all_data["YearRemodAdd"]
all_data["GarageYrBlt"] = all_data["GarageYrBlt"]
all_data["GarageYrBlt"].fillna(0.0, inplace=True)
 all_data["MoSold"] = all_data["MoSold"]
all_data["YrSold"] = all_data["YrSold"]
all_data["LowQualFinSF"] = all_data["LowQualFinSF"]
all_data["MiscVal"] = all_data["MiscVal"]
all_data["PoolQC"] = all_data["PoolQC"].apply(lambda x: qual_dict[x])
 all_data["PoolArea"] = all_data["PoolArea"]
all_data["PoolArea"].fillna(0, inplace=True )
gc.collect()<create_dataframe>
for ordinal in ['MSSubClass']:
all_data[ordinal] = all_data[ordinal].astype(str )
folds = KFold(n_splits=5, shuffle=True, random_state=15)
 oof = np.zeros(len(train))
categorical_columns = [c for c in categorical_columns if c not in ['MachineIdentifier']]
features = [c for c in train.columns if c not in ['MachineIdentifier']]
predictions = np.zeros(len(test))
start = time.time()
feature_importance_df = pd.DataFrame()
start_time= time.time()
score = [0 for _ in range(folds.n_splits)]
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(train.iloc[trn_idx][features],
label=target.iloc[trn_idx],
categorical_feature = categorical_columns 
)
 val_data = lgb.Dataset(train.iloc[val_idx][features],
label=target.iloc[val_idx],
categorical_feature = categorical_columns 
)
 num_round = 10000
clf = lgb.train(param,
trn_data,
num_round,
valid_sets = [trn_data, val_data],
verbose_eval=100,
early_stopping_rounds = 200)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance(importance_type='gain')
 fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 initial_idx = 0
chunk_size = 1000000
current_pred = np.zeros(len(test))
while initial_idx < test.shape[0]:
final_idx = min(initial_idx + chunk_size, test.shape[0])
 idx = range(initial_idx, final_idx)
 current_pred[idx] = clf.predict(test.iloc[idx][features], num_iteration=clf.best_iteration)
 initial_idx = final_idx
predictions += current_pred / min(folds.n_splits, max_iter)
 print("time elapsed: {:<5.2}s".format(( time.time() - start_time)/ 3600))
score[fold_] = metrics.roc_auc_score(target.iloc[val_idx], oof[val_idx])
 if fold_ == max_iter - 1: break
if(folds.n_splits == max_iter):
print("CV score: {:<8.5f}".format(metrics.roc_auc_score(target, oof)))
 else:
print("CV score: {:<8.5f}".format(sum(score)/ max_iter))<create_dataframe>
all_na =(all_data.isnull().sum() * 100)/ len(all_data)
 all_na = all_na.drop(all_na[all_na == 0].index ).sort_values(ascending=False)
 all_na
sub_df = pd.DataFrame({"MachineIdentifier": test["MachineIdentifier"].values})
 sub_df["HasDetections"] = predictions
sub_df[:10]<save_to_csv>
def fill_with_NA(df, column):
df[column] = df[column].fillna("NA")
 return df
sub_df.to_csv("submit.csv", index=False )<import_modules>
def fill_with_mode(df, column):
df[column] = df[column].fillna(df[column].mode() [0])
 return df
import statsmodels.api as sm
from statsmodels.sandbox.regression.predstd import wls_prediction_std
import math
import sklearn.preprocessing as skpe
import sklearn.model_selection as ms
import sklearn.metrics as sklm
import sklearn.ensemble as sken
import sklearn.linear_model as lm
import seaborn as sns
import matplotlib.pyplot as plt<load_from_csv>
all_data = fill_with_NA(all_data, 'PoolQC' )
path=".. /input/rossmann-store-sales/train.csv"
train=pd.read_csv(path)
 print(train.shape)
 train.head()<load_from_csv>
all_data = fill_with_NA(all_data, 'MiscFeature' )
path1=".. /input/rossmann-store-sales/test.csv"
test=pd.read_csv(path1)
 print(test.shape)
 test.head()<load_from_csv>
all_data = fill_with_NA(all_data, 'Alley' )
path2=".. /input/rossmann-store-sales/store.csv"
store_df=pd.read_csv(path2)
 print(store_df.shape)
 store_df.head()<feature_engineering>
all_data = fill_with_NA(all_data, 'Fence' )
train['Year'] = train['Date'].apply(lambda x: int(str(x)[:4]))
train['Month'] = train['Date'].apply(lambda x: int(str(x)[5:7]))
test['Year'] = test['Date'].apply(lambda x: int(str(x)[:4]))
test['Month'] = test['Date'].apply(lambda x: int(str(x)[5:7]))
train['Date'] = train['Date'].apply(lambda x:(str(x)[:7]))
test['Date'] = test['Date'].apply(lambda x:(str(x)[:7]))
avg_sales = train.groupby('Date')["Sales"].mean()
pct_change_sales = train.groupby('Date')["Sales"].sum().pct_change()
fig,(axis1,axis2)= plt.subplots(2,1,sharex=True,figsize=(15,8))
ax1 = avg_sales.plot(legend=True,ax=axis1,marker='o',title="Average Sales")
 ax1.set_xticks(range(len(avg_sales)))
 ax1.set_xticklabels(avg_sales.index.tolist() , rotation=90)
 ax2 = pct_change_sales.plot(legend=True,ax=axis2,marker='o',rot=90,colormap="summer",title="Sales Percent Change" )<count_values>
all_data = fill_with_NA(all_data, 'FireplaceQu' )
store_df['PromoInterval'].value_counts()<merge>
all_data["LotFrontage"] = all_data['LotFrontage'].fillna(all_data['LotFrontage'].median() )
train_store = pd.merge(right=store_df,left=train,how='outer',on='Store')
 train_store.head()<count_missing_values>
for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
all_data = fill_with_NA(all_data, col )
train_store.isnull().sum()<data_type_conversions>
for col in ['GarageYrBlt', 'GarageArea', 'GarageCars']:
all_data[col] = all_data[col].fillna(0 )
med_sales_per_customer = train_store['Sales_per_customer'].astype('float' ).median(axis=0)
 train_store['Sales_per_customer'].replace(np.nan,med_sales_per_customer,inplace=True)
 med_comp_dist = train_store['CompetitionDistance'].astype('float' ).median(axis=0)
 train_store['CompetitionDistance'].replace(np.nan,math.floor(med_comp_dist),inplace=True)
 med_comp_month = train_store['CompetitionOpenSinceMonth'].astype('float' ).median(axis=0)
 train_store['CompetitionOpenSinceMonth'].replace(np.nan,math.floor(med_comp_month),inplace=True)
 med_comp_year = train_store['CompetitionOpenSinceYear'].astype('float' ).median(axis=0)
 train_store['CompetitionOpenSinceYear'].replace(np.nan,math.floor(med_comp_year),inplace=True)
 med_promo2_week = train_store['Promo2SinceWeek'].astype('float' ).median(axis=0)
 train_store['Promo2SinceWeek'].replace(np.nan,math.floor(med_promo2_week),inplace=True)
 med_promo2_year = train_store['Promo2SinceYear'].astype('float' ).median(axis=0)
 train_store['Promo2SinceYear'].replace(np.nan,math.floor(med_promo2_year),inplace=True)
 train_store.drop(['PromoInterval'],axis=1,inplace=True )<data_type_conversions>
for col in ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath','MasVnrArea']:
all_data[col] = all_data[col].fillna(0 )
train.drop(['Customers','Sales_per_customer'],axis=1,inplace=True)
 train['Open'] = train['Open'].astype(float )<drop_column>
for col in ['BsmtFinType2','BsmtFinType1', 'BsmtExposure','BsmtCond','BsmtQual','Exterior2nd','Exterior1st','MasVnrType']:
all_data = fill_with_NA(all_data, col )
train.drop(['Year','Month','SchoolHoliday','Date'],axis=1,inplace=True)
 test.drop(['Year','Month','SchoolHoliday','Date'],axis=1,inplace=True )<categorify>
all_data = fill_with_mode(all_data, 'MSZoning' )
train_dummies = pd.get_dummies(train['DayOfWeek'], prefix='Day')
 train_dummies.drop(['Day_7'], axis=1, inplace=True)
 test_dummies = pd.get_dummies(test['DayOfWeek'],prefix='Day')
 test_dummies.drop(['Day_7'], axis=1, inplace=True)
 train = train.join(train_dummies)
 test = test.join(test_dummies )<data_type_conversions>
all_data = fill_with_mode(all_data, 'Functional' )
test["Open"][test["Open"] != test["Open"]] =(test["DayOfWeek"] != 7 ).astype(float )<drop_column>
all_data = fill_with_mode(all_data, 'Utilities' )
train.drop(['DayOfWeek'], axis=1,inplace=True)
 test.drop(['DayOfWeek'], axis=1,inplace=True)
 train= train[train_store["Open"] != 0]<filter>
all_data = fill_with_mode(all_data, 'SaleType' )
closed_ids = test["Id"][test["Open"] == 0].values<drop_column>
all_data = fill_with_mode(all_data, 'KitchenQual' )
test = test[test["Open"] != 0]<prepare_x_and_y>
all_data = fill_with_mode(all_data, 'Electrical' )
train_dic = dict(list(train.groupby('Store')))
 test_dic = dict(list(test.groupby('Store')))
 submission = pd.Series()
scores = []
for i in test_dic:
store = train_dic[i]
train_x = store.drop(["Sales","Store"],axis=1)
 train_y = store["Sales"]
test_x = test_dic[i].copy()
store_ids = test_x["Id"]
test_x.drop(["Id","Store"], axis=1,inplace=True)
 lr = lm.LinearRegression(normalize=True)
 lr.fit(train_x, train_y)
 pred_y = lr.predict(test_x)
 scores.append(lr.score(train_x, train_y))
submission = submission.append(pd.Series(pred_y, index=store_ids))
submission = submission.append(pd.Series(0, index=closed_ids))
submission = pd.DataFrame({ "Id": submission.index, "Sales": submission.values})
 submission.to_csv('rossmann.csv', index=False )<import_modules>
all_na =(all_data.isnull().sum() * 100)/ len(all_data)
 all_na = all_na.drop(all_na[all_na == 0].index ).sort_values(ascending=False)
 all_na
import pandas as pd
from pandas import datetime
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np<load_from_csv>
all_data['ExterQual'] = all_data['ExterQual'].fillna("NA" )
df = pd.read_csv('.. /input/rossmann-store-sales/train.csv', parse_dates = ['Date'], low_memory = False)
 df.head()<data_type_conversions>
all_data['TotalAreaSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
all_data['AllSF'] = all_data['TotalAreaSF'] + all_data['WoodDeckSF'] + all_data['TotalBsmtSF'] + all_data['OpenPorchSF']
df['Date']=pd.to_datetime(df['Date'],format='%Y-%m-%d' )<feature_engineering>
y_log = np.log1p(y )
df['Hour'] = df['Date'].dt.hour
df['Day_of_Month'] = df['Date'].dt.day
df['Day_of_Week'] = df['Date'].dt.dayofweek
df['Month'] = df['Date'].dt.month<load_from_csv>
skewed = set()
log_skewed = set()
for column in numerical_columns:
if abs(skew(all_data[column])) > 0.5:
skewed.add(column)
 if abs(skew(np.log1p(all_data[column])))> 0.5:
log_skewed.add(column)
 print("{} out of {} are skewed".format(len(skewed), len(numerical_columns)))
 print("{} are not log normal".format(len(log_skewed)))
 log_skewed
test = pd.read_csv('.. /input/rossmann-store-sales/test.csv', parse_dates = True, low_memory = False)
 test.head()<data_type_conversions>
for column in(skewed):
all_data[column] = boxcox1p(all_data[column], 0.15)
test['Date']=pd.to_datetime(test['Date'],format='%Y-%m-%d' )<feature_engineering>
for column in ['Alley',
'CentralAir',
'LandSlope',
'LotShape',
'MSSubClass',
'MoSold',
'OverallCond',
'PavedDrive',
'Street',
'YrSold',
'YearBuilt','GarageYrBlt',
]:
all_data[column] = LabelEncoder().fit_transform(all_data[column] )
test['Hour'] = test['Date'].dt.hour
test['Day_of_Month'] = test['Date'].dt.day
test['Day_of_Week'] = test['Date'].dt.dayofweek
test['Month'] = test['Date'].dt.month<count_values>
all_data = pd.get_dummies(all_data )
df.Open.value_counts()<count_values>
train = all_data[:train_size]
test = all_data[train_size:]
np.sum([df['Sales'] == 0] )<filter>
def rms_error(model):
return np.sqrt(-1 * cross_val_score(model, train.values, y_log.values, scoring="neg_mean_squared_error", cv = 5))
df = df[(df["Open"] != 0)&(df['Sales'] != 0)]<load_from_csv>
from sklearn.impute import SimpleImputer
from sklearn.pipeline import make_pipeline, Pipeline
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.preprocessing import StandardScaler, Normalizer, MinMaxScaler, RobustScaler
from sklearn.linear_model import ElasticNet, ElasticNetCV, SGDRegressor, HuberRegressor, LassoCV, RidgeCV, Lasso
from sklearn.svm import SVR
from sklearn.kernel_ridge import KernelRidge
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
store = pd.read_csv('.. /input/rossmann-store-sales/store.csv')
 store.head(30 )<count_missing_values>
pipeline = Pipeline([('model', RandomForestRegressor())])
 random_forest_param_grid = {
"model__n_estimators": [100,200, 300, 400, 500, 600, 700],
"model__max_leaf_nodes": [ 100, 200, 300, 400, 500, 600, 650, 700, 800]
}
random_forest_grid = GridSearchCV(pipeline, cv=5, param_grid=random_forest_param_grid, n_jobs=-1 )
store.isnull().sum()<correct_missing_values>
random_forest_grid.fit(train, y_log )
store['CompetitionDistance'] = store['CompetitionDistance'].fillna(store['CompetitionDistance'].max())
 store['CompetitionOpenSinceMonth'] = store['CompetitionOpenSinceMonth'].fillna(store['CompetitionOpenSinceMonth'].mode().iloc[0])
 store['CompetitionOpenSinceYear'] = store['CompetitionOpenSinceYear'].fillna(store['CompetitionOpenSinceYear'].mode().iloc[0])
 store['Promo2SinceWeek'] = store['Promo2SinceWeek'].fillna(0)
 store['Promo2SinceYear'] = store['Promo2SinceYear'].fillna(store['Promo2SinceYear'].mode().iloc[0])
 store['PromoInterval'] = store['PromoInterval'].fillna(store['PromoInterval'].mode().iloc[0])
 store.head()<merge>
random_forest_grid.best_params_
df_store = pd.merge(df, store, how = 'left', on = 'Store')
 df_store.head()<groupby>
pipeline = Pipeline([('imputer',SimpleImputer()),('model', GradientBoostingRegressor())])
 gradient_boosting_param_grid = {
"model__n_estimators": [ 500, 600,700,800,1000],
"model__max_depth": [2, 3, 4]
}
gradient_boosting_grid = GridSearchCV(pipeline, cv=5, param_grid=gradient_boosting_param_grid, n_jobs=-1 )
df_store.groupby('StoreType')['Customers', 'Sales'].sum()<categorify>
gradient_boosting_grid.fit(train, y_log )
df_store['StateHoliday'] = df_store['StateHoliday'].map({'0':0 , 0:0 , 'a':1 , 'b':2 , 'c':3})
 df_store['StateHoliday'] = df_store['StateHoliday'].astype(int )<categorify>
gradient_boosting_grid.best_params_
df_store['StoreType'] = df_store['StoreType'].map({'a':1 , 'b':2 , 'c':3 , 'd':4})
 df_store['StoreType'] = df_store['StoreType'].astype(int )<count_missing_values>
pipeline = Pipeline([('imputer',SimpleImputer()),('model', XGBRegressor())])
 param_grid = {
"model__n_estimators": [500,600,800, 850, 1000, 1200],
"model__learning_rate": [0.01, 0.02, 0.05,0.75, 0.1, 0.2]
}
fit_params = {"model__eval_set": [(train.as_matrix() , y_log.as_matrix())],
"model__early_stopping_rounds": 10,
"model__verbose": False}
xgb_grid = GridSearchCV(pipeline, cv=5, param_grid=param_grid, fit_params = fit_params, n_jobs=32 )
df_store.isnull().sum()<categorify>
xgb_grid.fit(train, y_log )
df_store['Assortment'] = df_store['Assortment'].map({'a':1 , 'b':2 , 'c':3})
 df_store['Assortment'] = df_store['Assortment'].astype(int )<categorify>
xgb_grid.best_params_
df_store['PromoInterval'] = df_store['PromoInterval'].map({'Jan,Apr,Jul,Oct':1 , 'Feb,May,Aug,Nov':2 , 'Mar,Jun,Sept,Dec':3})
 df_store['PromoInterval'] = df_store['PromoInterval'].astype(int )<save_to_csv>
pipeline_list = [
Pipeline([('scaler', RobustScaler()),('model', ElasticNetCV(alphas = [4, 1, 0.1, 0.001, 0.0005, 5e-4], l1_ratio = [1, 0.1, 0.001, 0.005])) ]),
Pipeline([('scaler', RobustScaler()),('model', RidgeCV(alphas = [10, 1, 0.1, 0.001, 0.0005, 5e-4])) ]),
Pipeline([('scaler', RobustScaler()),('model', LassoCV(alphas = [1, 0.1, 0.001, 0.0005, 5e-4], max_iter=50000)) ]),
Pipeline([('model', XGBRegressor(learning_rate=0.05, n_estimators=800)) ]),
Pipeline([('model', GradientBoostingRegressor(max_depth=2, n_estimators=600, loss='huber')) ]),
Pipeline([('scaler', RobustScaler()),('model', KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)) ]),
Pipeline([('model', LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=600,
max_bin = 50, bagging_fraction = 0.6,
bagging_freq = 5, feature_fraction = 0.25,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf = 6, min_sum_hessian_in_leaf = 11)) ])
 ]
model_names = ['elasticNet','ridge','lasso','XGB','GBR','KernelRidge','LGBM']
df_store.to_csv('df_merged.csv', index=False )<count_missing_values>
error_list =list()
for index, pipeline in enumerate(pipeline_list):
error_list.append(rms_error(pipeline ).mean() )
df_store.isnull().sum()<merge>
errors = pd.DataFrame({
'models': model_names,
'error':error_list
})
 errors.set_index('models', inplace=True)
 errors
test = pd.merge(test, store, how = 'left', on = 'Store')
 test.head()<count_missing_values>
multiple = pd.DataFrame()
for index, pipeline in enumerate(pipeline_list):
pipeline.fit(train, y_log)
 multiple[index] = pipeline.predict(test )
test.isnull().sum()<count_missing_values>
averaged_predictions = np.expm1(multiple.mean(axis=1))
test.isnull().sum()<correct_missing_values>
class StackingModel(BaseEstimator, TransformerMixin, RegressorMixin):
def __init__(self, base_models, meta_model):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = max(len(base_models), 2)
 def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 k_folds = KFold(n_splits = self.n_folds, shuffle=True, random_state = 46)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for ix, model in enumerate(self.base_models):
for train_ix, test_ix in k_folds.split(X, y):
m = clone(model)
 m.fit(X[train_ix], y[train_ix])
 self.base_models_[ix].append(m)
 y_predicted = m.predict(X[test_ix])
 out_of_fold_predictions[test_ix, ix] = y_predicted
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
return self.meta_model_.predict(
 np.column_stack(
 [
np.column_stack(model.predict(X)for model in models ).mean(axis=1)
 for models in self.base_models_
] 
)
 )
test.fillna(method='ffill', inplace=True )<categorify>
rms_error(StackingModel(base_models=pipeline_list, meta_model=Lasso(alpha =0.0005, random_state=1)) ).mean()
test['StateHoliday'] = test['StateHoliday'].map({'0':0 , 0:0 , 'a':1 , 'b':2 , 'c':3})
 test['StateHoliday'] = test['StateHoliday'].astype(int)
 test['StoreType'] = test['StoreType'].map({'a':1 , 'b':2 , 'c':3 , 'd':4})
 test['StoreType'] = test['StoreType'].astype(int)
 test['Assortment'] = test['Assortment'].map({'a':1 , 'b':2 , 'c':3})
 test['Assortment'] = test['Assortment'].astype(int)
 test['PromoInterval'] = test['PromoInterval'].map({'Jan,Apr,Jul,Oct':1 , 'Feb,May,Aug,Nov':2 , 'Mar,Jun,Sept,Dec':3})
 test['PromoInterval'] = test['PromoInterval'].astype(int )<save_to_csv>
stack = StackingModel(base_models=pipeline_list, meta_model=Lasso(alpha =0.0005, random_state=1))
stack.fit(train.values, y_log.values )
test.to_csv('test_merged.csv', index=False )<drop_column>
m_xgb = XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 m_xgb.fit(train.values, y_log.values )
test = test.drop(['Id','Date'],axis=1 )<split>
m_lgbm = LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 m_lgbm.fit(train.values, y_log.values )
X = df_store.drop(['Date','Sales','Customers', 'SalesPerCustomer'],1)
 y = np.log1p(df_store['Sales'])
 X_train , X_val , y_train , y_val = train_test_split(X, y , test_size=0.30 , random_state = 1 )<train_model>
answer = np.expm1(0.6 * stack.predict(test.values)+ 0.2 * m_xgb.predict(test.values)+ 0.2 * m_lgbm.predict(test.values))
gbrt = GradientBoostingRegressor(max_depth=10, n_estimators=200, random_state=42)
 gbrt.fit(X_train, y_train)
 print(gbrt.score(X_train, y_train))<predict_on_test>
submission = pd.DataFrame({'Id': test_ID, 'SalePrice':answer} )
y_pred = gbrt.predict(X_val )<compute_test_metric>
submission.to_csv('submission_stacked.csv', index=False )