Kaggle competition: Walmart Recruiting - Store Sales Forecasting
<SOS> metric: WMAE Kaggle data source: walmart-recruiting-store-sales-forecasting<import_modules>
warnings.filterwarnings("ignore")
 pysqldf = lambda q: sqldf(q, globals() )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
import seaborn as sns
from sklearn_pandas import CategoricalImputer
from sklearn.impute import SimpleImputer
from sklearn.decomposition import PCA
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PowerTransformer
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import PolynomialFeatures
from sklearn.svm import OneClassSVM
from sklearn.covariance import EllipticEnvelope
from scipy.stats import normaltest
from scipy.stats import shapiro
from scipy.stats import probplot
import os
import warnings<load_from_csv>
features = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/features.csv.zip')
 train = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/train.csv.zip')
 stores = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/stores.csv')
 test = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/test.csv.zip')
 sample_submission = pd.read_csv('.. /input/walmart-recruiting-store-sales-forecasting/sampleSubmission.csv.zip' )
print(os.listdir(".. /input"))
df = pd.read_csv('.. /input/train.csv', delimiter=',')
 test = pd.read_csv('.. /input/test.csv', delimiter=',')
 nRow, nCol = df.shape
print(f'Train dataset has {nRow} rows and {nCol} columns')
 nRow, nCol = test.shape
print(f'Test dataset has {nRow} rows and {nCol} columns')
 df.head(20 )<drop_column>
feat_sto = features.merge(stores, how='inner', on='Store' )
train = df.sample(frac=0.8, random_state=100)
 dev = df.drop(train.index )<compute_test_metric>
pd.DataFrame(feat_sto.dtypes, columns=['Type'] )
def hypothesis(Theta, X):
m = X.shape[0]
yPred = np.zeros(( m,1))
X = np.column_stack(( np.ones(( m, 1)) , X))
yPred = np.matmul(X, Theta)
 return yPred
def costFunction(Theta, X, y, func=hypothesis):
m = y.shape[0]
J = 0
yPred = func(Theta, X)
 J = np.sum(np.square(yPred - y)) /(2 * m)
 return J<compute_test_metric>
pd.DataFrame({'Type_Train': train.dtypes, 'Type_Test': test.dtypes} )
function = lambda x: np.power(x, 2)
 derivative = lambda x: x*2
gradientExplanation(function, derivative )<compute_test_metric>
feat_sto.Date = pd.to_datetime(feat_sto.Date)
 train.Date = pd.to_datetime(train.Date)
 test.Date = pd.to_datetime(test.Date )
learningRateExplanation(function, derivative )<compute_test_metric>
feat_sto['Week'] = feat_sto.Date.dt.week
feat_sto['Year'] = feat_sto.Date.dt.year
def gradients(Theta, X, y):
m = y.shape[0]
grads = np.zeros(Theta.shape)
 X = np.column_stack(( np.ones(( m,1)) , X))
grads = np.matmul(X.transpose() , np.matmul(X, Theta)- y)/ m
return grads
def gradientDescent(Theta, X, y, learning_rate=0.00000001, iterations=1000, gradFunc=gradients, costFunc=costFunction):
m = y.shape[0]
Js = [costFunc(Theta, X, y, hypothesis)]
Thetas = [Theta]
for i in range(iterations):
grads = gradFunc(Theta, X, y)
 Theta = Theta - learning_rate * grads
Js.append(costFunc(Theta, X, y, hypothesis))
Thetas.append(Theta)
 return(Js, Theta, Thetas )<compute_test_metric>
train_detail = train.merge(feat_sto,
how='inner',
on=['Store','Date','IsHoliday'] ).sort_values(by=['Store',
'Dept',
'Date'] ).reset_index(drop=True )
def costFunctionRegularized(Theta, X, y, func=hypothesis, lam=0.01):
m = y.shape[0]
J = 0
yPred = func(Theta, X)
 J =(np.sum(np.square(yPred - y)) + lam * np.sum(np.square(Theta[1:])))/(2 * m)
 return J<compute_test_metric>
test_detail = test.merge(feat_sto,
how='inner',
on=['Store','Date','IsHoliday'] ).sort_values(by=['Store',
'Dept',
'Date'] ).reset_index(drop=True )
def gradientsRegularized(Theta, X, y, lam=0.01):
m = y.shape[0]
grads = np.zeros(Theta.shape)
 reg = np.ones(Theta.shape)* lam
reg[0] = 0
X = np.column_stack(( np.ones(( m,1)) , X))
grads =(np.matmul(X.transpose() , np.matmul(X, Theta)- y)+ reg * Theta)/ m
return grads<drop_column>
del features, train, stores, test
train_data = df.drop(['SalePrice', 'Id'], axis=1)
 train_output = df['SalePrice']
test_data = test.drop('Id', axis=1 )<split>
null_columns =(train_detail.isnull().sum(axis = 0)/len(train_detail)).sort_values(ascending=False ).index
null_data = pd.concat([
train_detail.isnull().sum(axis = 0), 
(train_detail.isnull().sum(axis = 0)/len(train_detail)).sort_values(ascending=False),
train_detail.loc[:, train_detail.columns.isin(list(null_columns)) ].dtypes], axis=1)
 null_data = null_data.rename(columns={0: '
1: '% null',
2: 'type'} ).sort_values(ascending=False, by = '% null')
 null_data = null_data[null_data["
null_data
identifier = lambda x: x
scores = {}
def train_dev_split(data, frac = 0.8):
train = data.sample(frac=frac, random_state=3)
 train_indexes = data.index.isin(train.index.values)
 dev = data.drop(train.index.values, axis=0)
 dev_indexes = data.index.isin(dev.index.values)
 return(train_indexes, dev_indexes)
 def rmsle(real, predicted):
s=0.0
for x in range(len(predicted)) :
p = np.log(predicted[x]+1)
 r = np.log(real[x]+1)
 s = s +(p - r)**2
return(s/len(predicted)) **0.5
def score(model, data, data_selector, is_rmsle=False):
y, X, _, train_ids, dev_ids = data
dev_s =.0
train_s =.0
if X is not None:
d = data_selector(y, X)
 if d.shape[1] > 0:
X_train, X_dev, y_train, y_dev = d[train_ids], d[dev_ids], y[train_ids], y[dev_ids]
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 model.fit(X_train, y_train)
 if is_rmsle:
pred = model.predict(X_dev)
 dev_s = rmsle(y_dev.values, pred)
 pred = model.predict(X_train)
 train_s = rmsle(y_train.values, pred)
 else:
dev_s = model.score(X_dev, y_dev)
 train_s = model.score(X_train, y_train)
 return dev_s, train_s
def validateModels(data, models, new_selector, data_transformer=identifier, is_rmsle=False):
new_data = data_transformer(data)
 new_models = {}
for key,(model, selector)in models.items() :
s1, t1 =(0, 0)
 s2, t2 =(0, 0)
 if selector is not None:
s1, t1 = score(model, data, selector, is_rmsle)
 s2, t2 = score(model, new_data, selector, is_rmsle)
 s3, t3 = score(model, new_data, new_selector, is_rmsle)
 if s2 >= s3:
new_models[key] =(model, selector)
 scores[key] = s2
print(f'{key} = dev accuracy {s2} - change by {s2 - s1}, train accuracy {t2}')
 else:
new_models[key] =(model, new_selector)
 scores[key] = s3
print(f'{key} = dev accuracy {s3} - change by {s3 - s1}, train accuracy {t3}')
 return(new_data, new_models)
 def submit_model(data, model, name=''):
alg, selector = model
train_output, train_data, test_data, _, _ = data
train_data = selector(train_output, train_data)
 alg.fit(train_data, train_output)
 pred = alg.predict(test_data)
 my_submission = pd.DataFrame({'Id': test.Id.values, 'SalePrice': np.expm1(pred ).reshape(-1)})
 my_submission.to_csv(name + 'submission.csv', index=False )<split>
train_detail.loc[(train_detail.Year==2010)&(train_detail.Week==13), 'IsHoliday'] = True
train_detail.loc[(train_detail.Year==2011)&(train_detail.Week==16), 'IsHoliday'] = True
train_detail.loc[(train_detail.Year==2012)&(train_detail.Week==14), 'IsHoliday'] = True
test_detail.loc[(test_detail.Year==2013)&(test_detail.Week==13), 'IsHoliday'] = True
models = {
'linear_regression' :(LinearRegression(n_jobs=-1), None)
 }
def chooser(y, X):
d = X.dropna(axis=1)
 series = d.corrwith(y)
 high_corr_columns = set(series[series > 0.5].index.values)
 number_columns = set(d.select_dtypes(include=['number'] ).columns.values)
 cols = list(high_corr_columns.intersection(number_columns))
return d[cols]
def setter(dataset):
train_rows, dev_rows = train_dev_split(train_data)
 return(train_output.copy() , train_data.copy() , test_data.copy() , train_rows, dev_rows)
 train_ds, dev_ids = train_dev_split(train_data)
 data =(None, None, None, None)
 data, models = validateModels(data, models, chooser, setter )<sort_values>
train_detail = train_detail.drop(columns=['Fuel_Price','MarkDown1','MarkDown2','MarkDown3','MarkDown4','MarkDown5'])
 test_detail = test_detail.drop(columns=['Fuel_Price','MarkDown1','MarkDown2','MarkDown3','MarkDown4','MarkDown5'] )
def missing_data(data):
total = data.isnull().sum().sort_values(ascending=False)
 percent =(data.isnull().sum() /data.isnull().count() ).sort_values(ascending=False)
 md = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 return md
missing_train_data = missing_data(pd.concat([df, test], sort=True, axis=0))
missing_train_data[missing_train_data['Total']>0]<categorify>
train_detail.Type = train_detail.Type.apply(lambda x: 3 if x == 'A' else(2 if x == 'B' else 1))
test_detail.Type = test_detail.Type.apply(lambda x: 3 if x == 'A' else(2 if x == 'B' else 1))
def manual_cleaner(dataset):
def build_impute_map(data, missing, number, def_value):
m = {}
if number:
m = dict.fromkeys(set(data.select_dtypes(include=['number'])) &
set(missing[missing.Percent >.0].index.values), def_value)
 else:
m = dict.fromkeys(set(data.select_dtypes(exclude=['number'])) &
set(missing[missing.Percent >.0].index.values), def_value)
 return m
def man_impute(mappings, data):
for k, v in mappings.items() :
data[k].fillna(v, inplace=True)
 return data
def impute(data, missing):
numeric_map = build_impute_map(data, missing, True, 0)
 data = man_impute(numeric_map, data)
 categorical_map = build_impute_map(data, missing, False, 'None')
 categorical_map['MasVnrType'] = 'None'
categorical_map['Electrical'] = 'SBrkr'
categorical_map['Utilities'] = 'AllPub'
data = man_impute(categorical_map, data)
 return data
df_man = dataset[1].copy()
test_man = dataset[2].copy()
df_man = impute(df_man, missing_train_data)
 test_man = impute(test_man, missing_train_data)
 test_man.fillna(0, inplace=True)
 return(dataset[0], df_man, test_man, dataset[3], dataset[4])
<categorify>
train_detail = train_detail.drop(columns=['Temperature'])
 test_detail = test_detail.drop(columns=['Temperature'] )
def automatic_cleaner(dataset):
def build_impute_list(data, number):
m = {}
if number:
m = [data.select_dtypes(include=['number'] ).columns.values]
else:
m = data.select_dtypes(exclude=['number'] ).columns.values
return m
def auto_impute(imputer, cols, train, test):
for col in cols:
train[col] = imputer.fit_transform(train[col])
 test[col] = imputer.transform(test[col])
 return(train, test)
 df_aut = dataset[1].copy()
test_aut = dataset[2].copy()
num_cols = build_impute_list(df_aut, True)
 cat_cols = build_impute_list(df_aut, False)
 num_imputer = SimpleImputer()
cat_imputer = CategoricalImputer()
df_aut, test_aut = auto_impute(cat_imputer, cat_cols, df_aut, test_aut)
 df_aut, test_aut = auto_impute(num_imputer, num_cols, df_aut, test_aut)
 test_aut.fillna(0, inplace=True)
 return(dataset[0], df_aut, test_aut, dataset[3], dataset[4] )<prepare_output>
train_detail = train_detail.drop(columns=['CPI'])
 test_detail = test_detail.drop(columns=['CPI'] )
data = aut_data
models = aut_models<categorify>
train_detail = train_detail.drop(columns=['Unemployment'])
 test_detail = test_detail.drop(columns=['Unemployment'] )
def aut_categorical_filler(dataset):
def transform(cols, train, test):
avg = train.copy().drop([523, 1298], axis=0)['SqFeetPrice'].mean()
transformer = {}
for col in cols:
transformer[col] = train.groupby(col)['SqFeetPrice'].median().map(lambda x:(x - avg)/ avg ).to_dict()
for col in cols:
train[col] = train[col].map(lambda cat: transformer[col].get(cat, 0))
test[col] = test[col].map(lambda cat: transformer[col].get(cat, 0))
return(train, test)
 train_data = dataset[1].copy()
test_data = dataset[2].copy()
train_data['Size'] =(train_data['GrLivArea'] / 600 + 1 ).astype(int)
 test_data['Size'] =(test_data['GrLivArea'] / 600 + 1 ).astype(int)
 train_data['SqFeetPrice'] = train_output.values / train_data['GrLivArea'].values
categorical_columns = list(train_data.select_dtypes(exclude=['number'] ).columns.values)
 categorical_columns.extend(['MoSold','YrSold','YearBuilt','YearRemodAdd'])
 train_data, test_data = transform(categorical_columns, train_data, test_data)
 train_data.drop(['SqFeetPrice'], axis=1, inplace=True)
 return(dataset[0], train_data, test_data, dataset[3], dataset[4])
 def all_selector(y, X):
return X 
<data_type_conversions>
def WMAE(dataset, real, predicted):
weights = dataset.IsHoliday.apply(lambda x: 5 if x else 1)
 return np.round(np.sum(weights*abs(real-predicted)) /(np.sum(weights)) , 2 )
def mix_aut_categorical_filler(dataset):
def man_transform(dataset):
dataset["BsmtCond"] = dataset["BsmtCond"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["BsmtExposure"] = dataset["BsmtExposure"].astype(pd.api.types.CategoricalDtype(categories = ['None','Mn','Av','Gd'],ordered=True)).cat.codes
dataset["BsmtFinType1"] = dataset["BsmtFinType1"].astype(pd.api.types.CategoricalDtype(categories = ['None','Unf','LwQ','Rec','BLQ','ALQ','GLQ'],ordered=True)).cat.codes
dataset["BsmtFinType2"] = dataset["BsmtFinType2"].astype(pd.api.types.CategoricalDtype(categories = ['None','Unf','LwQ','Rec','BLQ','ALQ','GLQ'],ordered=True)).cat.codes
dataset["BsmtQual"] = dataset["BsmtQual"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["ExterCond"] = dataset["ExterCond"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["ExterQual"] = dataset["ExterQual"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["Fence"] = dataset["Fence"].astype(pd.api.types.CategoricalDtype(categories = ['None','MnWw','GdWo','MnPrv','GdPrv'],ordered=True)).cat.codes
dataset["FireplaceQu"] = dataset["FireplaceQu"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["Functional"] = dataset["Functional"].astype(pd.api.types.CategoricalDtype(categories = ['Sal','Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],ordered=True)).cat.codes
dataset["GarageCond"] = dataset["GarageCond"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["GarageFinish"] = dataset["GarageFinish"].astype(pd.api.types.CategoricalDtype(categories = ['None','Unf','RFn','Fin'],ordered=True)).cat.codes
dataset["GarageQual"] = dataset["GarageQual"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["HeatingQC"] = dataset["HeatingQC"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["KitchenQual"] = dataset["KitchenQual"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["PavedDrive"] = dataset["PavedDrive"].astype(pd.api.types.CategoricalDtype(categories = ['N','P','Y'],ordered=True)).cat.codes
dataset["PoolQC"] = dataset["PoolQC"].astype(pd.api.types.CategoricalDtype(categories = ['None','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["Utilities"] = dataset["Utilities"].astype(pd.api.types.CategoricalDtype(categories = ['ELO','NoSeWa','NoSewr','AllPub'],ordered=True)).cat.codes
def aut_transform(cols, train, test):
avg = train['SqFeetPrice'].mean()
transformer = {}
for col in cols:
transformer[col] = train.groupby(col)['SqFeetPrice'].mean().map(lambda x:(x - avg)/ avg ).to_dict()
for col in cols:
train[col] = train[col].map(lambda cat: transformer[col].get(cat, 0))
test[col] = test[col].map(lambda cat: transformer[col].get(cat, 0))
return(train, test)
 train_data = dataset[1].copy()
test_data = dataset[2].copy()
train_data['SqFeetPrice'] = train_output.values / train_data['GrLivArea'].values
man_transform(train_data)
 man_transform(test_data)
 categorical_columns = list(train_data.select_dtypes(exclude=['number'] ).columns.values)
 categorical_columns.extend(['MoSold','YrSold','YearBuilt','YearRemodAdd'])
 train_data, test_data = aut_transform(categorical_columns, train_data, test_data)
 train_data.drop(['SqFeetPrice'], axis=1, inplace=True)
 return(dataset[0], train_data, test_data, dataset[3], dataset[4])
<data_type_conversions>
def random_forest(n_estimators, max_depth):
result = []
for estimator in n_estimators:
for depth in max_depth:
wmaes_cv = []
for i in range(1,5):
print('k:', i, ', n_estimators:', estimator, ', max_depth:', depth)
 x_train, x_test, y_train, y_test = train_test_split(X_train, Y_train, test_size=0.3)
 RF = RandomForestRegressor(n_estimators=estimator, max_depth=depth)
 RF.fit(x_train, y_train)
 predicted = RF.predict(x_test)
 wmaes_cv.append(WMAE(x_test, y_test, predicted))
print('WMAE:', np.mean(wmaes_cv))
result.append({'Max_Depth': depth, 'Estimators': estimator, 'WMAE': np.mean(wmaes_cv)})
 return pd.DataFrame(result )
def mix_one_categorical_filler(dataset):
def man_transform(dataset):
dataset["BsmtCond"] = dataset["BsmtCond"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["BsmtExposure"] = dataset["BsmtExposure"].astype(pd.api.types.CategoricalDtype(categories = ['None','Mn','Av','Gd'],ordered=True)).cat.codes
dataset["BsmtFinType1"] = dataset["BsmtFinType1"].astype(pd.api.types.CategoricalDtype(categories = ['None','Unf','LwQ','Rec','BLQ','ALQ','GLQ'],ordered=True)).cat.codes
dataset["BsmtFinType2"] = dataset["BsmtFinType2"].astype(pd.api.types.CategoricalDtype(categories = ['None','Unf','LwQ','Rec','BLQ','ALQ','GLQ'],ordered=True)).cat.codes
dataset["BsmtQual"] = dataset["BsmtQual"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["ExterCond"] = dataset["ExterCond"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["ExterQual"] = dataset["ExterQual"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["Fence"] = dataset["Fence"].astype(pd.api.types.CategoricalDtype(categories = ['None','MnWw','GdWo','MnPrv','GdPrv'],ordered=True)).cat.codes
dataset["FireplaceQu"] = dataset["FireplaceQu"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["Functional"] = dataset["Functional"].astype(pd.api.types.CategoricalDtype(categories = ['Sal','Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],ordered=True)).cat.codes
dataset["GarageCond"] = dataset["GarageCond"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["GarageFinish"] = dataset["GarageFinish"].astype(pd.api.types.CategoricalDtype(categories = ['None','Unf','RFn','Fin'],ordered=True)).cat.codes
dataset["GarageQual"] = dataset["GarageQual"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["HeatingQC"] = dataset["HeatingQC"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["KitchenQual"] = dataset["KitchenQual"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["PavedDrive"] = dataset["PavedDrive"].astype(pd.api.types.CategoricalDtype(categories = ['N','P','Y'],ordered=True)).cat.codes
dataset["PoolQC"] = dataset["PoolQC"].astype(pd.api.types.CategoricalDtype(categories = ['None','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["Utilities"] = dataset["Utilities"].astype(pd.api.types.CategoricalDtype(categories = ['ELO','NoSeWa','NoSewr','AllPub'],ordered=True)).cat.codes
def one_transform(cols, train, test):
union = pd.get_dummies(pd.concat([train, test], sort=False), columns=cols)
 l_train = len(train)
 train, test = union[:l_train], union[l_train:]
return(train, test)
 train_data = dataset[1].copy()
test_data = dataset[2].copy()
man_transform(train_data)
 man_transform(test_data)
 categorical_columns = list(train_data.select_dtypes(exclude=['number'] ).columns.values)
 categorical_columns.extend(['MoSold','YrSold','YearBuilt','YearRemodAdd'])
 train_data, test_data = one_transform(categorical_columns, train_data, test_data)
 return(dataset[0], train_data, test_data, dataset[3], dataset[4] )<categorify>
def random_forest_II(n_estimators, max_depth, max_features):
result = []
for feature in max_features:
wmaes_cv = []
for i in range(1,5):
print('k:', i, ', max_features:', feature)
 x_train, x_test, y_train, y_test = train_test_split(X_train, Y_train, test_size=0.3)
 RF = RandomForestRegressor(n_estimators=n_estimators, max_depth=max_depth, max_features=feature)
 RF.fit(x_train, y_train)
 predicted = RF.predict(x_test)
 wmaes_cv.append(WMAE(x_test, y_test, predicted))
print('WMAE:', np.mean(wmaes_cv))
result.append({'Max_Feature': feature, 'WMAE': np.mean(wmaes_cv)})
 return pd.DataFrame(result )
def one_categorical_filler(dataset):
def transform(cols, train, test):
union = pd.get_dummies(pd.concat([train, test], sort=False), columns=cols)
 l_train = len(train)
 train, test = union[:l_train], union[l_train:]
return(train, test)
 train_data = dataset[1].copy()
test_data = dataset[2].copy()
categorical_columns = list(train_data.select_dtypes(exclude=['number'] ).columns.values)
 categorical_columns.extend(['MoSold','YrSold','YearBuilt','YearRemodAdd'])
 train_data, test_data = transform(categorical_columns, train_data, test_data)
 return(dataset[0], train_data, test_data, dataset[3], dataset[4] )<categorify>
def random_forest_III(n_estimators, max_depth, max_features, min_samples_split, min_samples_leaf):
result = []
for split in min_samples_split:
for leaf in min_samples_leaf:
wmaes_cv = []
for i in range(1,5):
print('k:', i, ', min_samples_split:', split, ', min_samples_leaf:', leaf)
 x_train, x_test, y_train, y_test = train_test_split(X_train, Y_train, test_size=0.3)
 RF = RandomForestRegressor(n_estimators=n_estimators, max_depth=max_depth, max_features=max_features,
min_samples_leaf=leaf, min_samples_split=split)
 RF.fit(x_train, y_train)
 predicted = RF.predict(x_test)
 wmaes_cv.append(WMAE(x_test, y_test, predicted))
print('WMAE:', np.mean(wmaes_cv))
result.append({'Min_Samples_Leaf': leaf, 'Min_Samples_Split': split, 'WMAE': np.mean(wmaes_cv)})
 return pd.DataFrame(result )
print('Automatically imputed dataset')
 aut_data, aut_models = validateModels(data, models, all_selector, aut_categorical_filler)
 print('Mix: manually and automatically imputed dataset')
 mix_aut_data, mix_aut_models = validateModels(data, models, all_selector, mix_aut_categorical_filler)
 print('Mix: manually imputed and one hot encoded dataset')
 mix_one_data, mix_one_models = validateModels(data, models, all_selector, mix_one_categorical_filler)
 print('One hot encoded dataset')
 one_data, one_models = validateModels(data, models, all_selector, one_categorical_filler )<load_pretrained>
X_train = train_detail[['Store','Dept','IsHoliday','Size','Week','Type','Year']]
Y_train = train_detail['Weekly_Sales']
data, models = aut_data, aut_models<normalization>
n_estimators = [56, 58, 60]
max_depth = [25, 27, 30]
random_forest(n_estimators, max_depth )
def standarizer(dataset):
train_data = dataset[1].copy()
test_data = dataset[2].copy()
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 X_scaler = RobustScaler()
train_data = pd.DataFrame(X_scaler.fit_transform(train_data), columns=train_data.columns.values)
 test_data = pd.DataFrame(X_scaler.transform(test_data), columns=test_data.columns.values)
 return(dataset[0], train_data, test_data, dataset[3], dataset[4])
 new_data, new_models = validateModels(data, models, all_selector, standarizer )<prepare_output>
min_samples_split = [2, 3, 4]
min_samples_leaf = [1, 2, 3]
random_forest_III(n_estimators=58, max_depth=27, max_features=6,
min_samples_split=min_samples_split, min_samples_leaf=min_samples_leaf )
data, models = new_data, new_models<sort_values>
RF = RandomForestRegressor(n_estimators=58, max_depth=27, max_features=6, min_samples_split=3, min_samples_leaf=1)
 RF.fit(X_train, Y_train )
correlation_matrix = train_data.corr()
correlation_pairs = correlation_matrix.unstack()
correlation_pairs[(correlation_pairs < 1.0)&(correlation_pairs >.8)].sort_values(kind="quicksort", ascending=False )<sort_values>
X_test = test_detail[['Store', 'Dept', 'IsHoliday', 'Size', 'Week', 'Type', 'Year']]
predict = RF.predict(X_test )
train_data.corrwith(train_output ).sort_values(ascending=False ).head(50 )<drop_column>
Final = X_test[['Store', 'Dept', 'Week']]
Final['Weekly_Sales'] = predict
def corr_features_remover(dataset):
cols_to_drop = ['Exterior1st', 'SaleCondition', 'GarageCars', 'GarageCond', 'TotRmsAbvGrd', 'GarageYrBlt', 'TotalBsmtSF']
return(dataset[0], dataset[1].drop(cols_to_drop, axis=1), dataset[2].drop(cols_to_drop, axis=1), dataset[3], dataset[4])
 man_data, man_models = validateModels(data, models, all_selector, corr_features_remover )<load_pretrained>
sample_submission['Weekly_Sales'] = Final_adj['Weekly_Sales_Adjusted']
sample_submission.to_csv('submission.csv',index=False )