Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_pretrained>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
import seaborn as sns
from sklearn_pandas import CategoricalImputer
from sklearn.impute import SimpleImputer
from sklearn.decomposition import PCA
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PowerTransformer
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import PolynomialFeatures
from sklearn.svm import OneClassSVM
from sklearn.covariance import EllipticEnvelope
from scipy.stats import normaltest
from scipy.stats import shapiro
from scipy.stats import probplot
import os
import warnings
xgbm = load(".. /input/xgboost-model/xgb_model.joblib")
 lgbm = load(".. /input/lightgbm-model/lightgbm_model.joblib")
 rfm = load(".. /input/randomforest-model/rf_model.joblib" )<load_from_csv>
print(os.listdir(".. /input"))
df = pd.read_csv('.. /input/train.csv', delimiter=',')
 test = pd.read_csv('.. /input/test.csv', delimiter=',')
 nRow, nCol = df.shape
print(f'Train dataset has {nRow} rows and {nCol} columns')
 nRow, nCol = test.shape
print(f'Test dataset has {nRow} rows and {nCol} columns')
 df.head(20 )
xgb_train = pd.read_csv('.. /input/xgboost-model/xgb_y_train.csv' ).set_index('ID')
 xgb_valid = pd.read_csv('.. /input/xgboost-model/xgb_y_valid.csv' ).set_index('ID')
 xgb_test = pd.read_csv('.. /input/xgboost-model/xgb_submission.csv' ).set_index('ID')
 lgb_train = pd.read_csv('.. /input/lightgbm-model/gbm_y_train.csv' ).set_index('ID')
 lgb_valid = pd.read_csv('.. /input/lightgbm-model/gbm_y_valid.csv' ).set_index('ID')
 lgb_test = pd.read_csv('.. /input/lightgbm-model/gbm_y_test.csv' ).set_index('ID')
 rf_train = pd.read_csv('.. /input/randomforest-model/rf_y_train.csv' ).set_index('ID')
 rf_valid = pd.read_csv('.. /input/randomforest-model/rf_y_valid.csv' ).set_index('ID')
 rf_test = pd.read_csv('.. /input/randomforest-model/rf_y_test.csv' ).set_index('ID')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
<prepare_x_and_y>
train = df.sample(frac=0.8, random_state=100)
 dev = df.drop(train.index )
X_train = np.hstack([xgb_train.values, lgb_train.values, rf_train.values])
 X_valid = np.hstack([xgb_valid.values, lgb_valid.values, rf_valid.values])
 X_test = np.hstack([xgb_test.values, lgb_test.values, rf_test.values] )<prepare_x_and_y>
def hypothesis(Theta, X):
m = X.shape[0]
yPred = np.zeros(( m,1))
X = np.column_stack(( np.ones(( m, 1)) , X))
yPred = np.matmul(X, Theta)
 return yPred
def costFunction(Theta, X, y, func=hypothesis):
m = y.shape[0]
J = 0
yPred = func(Theta, X)
 J = np.sum(np.square(yPred - y)) /(2 * m)
 return J
data = pd.read_pickle('.. /input/eda-preprocessing-feature-engineering/all_data.pkl')
 data = data[data.date_block_num > 5]
Y_train = data[data.date_block_num < 33]['item_cnt_month']
Y_valid = data[data.date_block_num == 33]['item_cnt_month']
del data
gc.collect()<train_model>
function = lambda x: np.power(x, 2)
 derivative = lambda x: x*2
gradientExplanation(function, derivative )
start_time = time.time()
lr = LinearRegression(n_jobs=-1, normalize=True)
 lr.fit(X_valid, Y_valid)
 print(f"Training took {time.time() - start_time} s")
 start_time = time.time()
Y_valid_pred = lr.predict(X_valid ).clip(0, 20)
 print(f"Predicting on valid set took {time.time() - start_time} s")
 print(f"VALID RMSE: {round(np.sqrt(mean_squared_error(Y_valid, Y_valid_pred)) , 5)}")
 dump(lr, 'stacking_model.joblib' )<save_to_csv>
learningRateExplanation(function, derivative )
start_time = time.time()
Y_test = lr.predict(X_test ).clip(0, 20)
 print(f"Predicting test set took {time.time() - start_time} s")
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('submission.csv', index=False)
<load_from_csv>
def gradients(Theta, X, y):
m = y.shape[0]
grads = np.zeros(Theta.shape)
 X = np.column_stack(( np.ones(( m,1)) , X))
grads = np.matmul(X.transpose() , np.matmul(X, Theta)- y)/ m
return grads
def gradientDescent(Theta, X, y, learning_rate=0.00000001, iterations=1000, gradFunc=gradients, costFunc=costFunction):
m = y.shape[0]
Js = [costFunc(Theta, X, y, hypothesis)]
Thetas = [Theta]
for i in range(iterations):
grads = gradFunc(Theta, X, y)
 Theta = Theta - learning_rate * grads
Js.append(costFunc(Theta, X, y, hypothesis))
Thetas.append(Theta)
 return(Js, Theta, Thetas )
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )<filter>
def costFunctionRegularized(Theta, X, y, func=hypothesis, lam=0.01):
m = y.shape[0]
J = 0
yPred = func(Theta, X)
 J =(np.sum(np.square(yPred - y)) + lam * np.sum(np.square(Theta[1:])))/(2 * m)
 return J
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]<feature_engineering>
def gradientsRegularized(Theta, X, y, lam=0.01):
m = y.shape[0]
grads = np.zeros(Theta.shape)
 reg = np.ones(Theta.shape)* lam
reg[0] = 0
X = np.column_stack(( np.ones(( m,1)) , X))
grads =(np.matmul(X.transpose() , np.matmul(X, Theta)- y)+ reg * Theta)/ m
return grads
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median<feature_engineering>
train_data = df.drop(['SalePrice', 'Id'], axis=1)
 train_output = df['SalePrice']
test_data = test.drop('Id', axis=1 )
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<categorify>
identifier = lambda x: x
scores = {}
def train_dev_split(data, frac = 0.8):
train = data.sample(frac=frac, random_state=3)
 train_indexes = data.index.isin(train.index.values)
 dev = data.drop(train.index.values, axis=0)
 dev_indexes = data.index.isin(dev.index.values)
 return(train_indexes, dev_indexes)
 def rmsle(real, predicted):
s=0.0
for x in range(len(predicted)) :
p = np.log(predicted[x]+1)
 r = np.log(real[x]+1)
 s = s +(p - r)**2
return(s/len(predicted)) **0.5
def score(model, data, data_selector, is_rmsle=False):
y, X, _, train_ids, dev_ids = data
dev_s =.0
train_s =.0
if X is not None:
d = data_selector(y, X)
 if d.shape[1] > 0:
X_train, X_dev, y_train, y_dev = d[train_ids], d[dev_ids], y[train_ids], y[dev_ids]
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 model.fit(X_train, y_train)
 if is_rmsle:
pred = model.predict(X_dev)
 dev_s = rmsle(y_dev.values, pred)
 pred = model.predict(X_train)
 train_s = rmsle(y_train.values, pred)
 else:
dev_s = model.score(X_dev, y_dev)
 train_s = model.score(X_train, y_train)
 return dev_s, train_s
def validateModels(data, models, new_selector, data_transformer=identifier, is_rmsle=False):
new_data = data_transformer(data)
 new_models = {}
for key,(model, selector)in models.items() :
s1, t1 =(0, 0)
 s2, t2 =(0, 0)
 if selector is not None:
s1, t1 = score(model, data, selector, is_rmsle)
 s2, t2 = score(model, new_data, selector, is_rmsle)
 s3, t3 = score(model, new_data, new_selector, is_rmsle)
 if s2 >= s3:
new_models[key] =(model, selector)
 scores[key] = s2
print(f'{key} = dev accuracy {s2} - change by {s2 - s1}, train accuracy {t2}')
 else:
new_models[key] =(model, new_selector)
 scores[key] = s3
print(f'{key} = dev accuracy {s3} - change by {s3 - s1}, train accuracy {t3}')
 return(new_data, new_models)
 def submit_model(data, model, name=''):
alg, selector = model
train_output, train_data, test_data, _, _ = data
train_data = selector(train_output, train_data)
 alg.fit(train_data, train_output)
 pred = alg.predict(test_data)
 my_submission = pd.DataFrame({'Id': test.Id.values, 'SalePrice': np.expm1(pred ).reshape(-1)})
 my_submission.to_csv(name + 'submission.csv', index=False )
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )<concatenate>
models = {
'linear_regression' :(LinearRegression(n_jobs=-1), None)
 }
def chooser(y, X):
d = X.dropna(axis=1)
 series = d.corrwith(y)
 high_corr_columns = set(series[series > 0.5].index.values)
 number_columns = set(d.select_dtypes(include=['number'] ).columns.values)
 cols = list(high_corr_columns.intersection(number_columns))
return d[cols]
def setter(dataset):
train_rows, dev_rows = train_dev_split(train_data)
 return(train_output.copy() , train_data.copy() , test_data.copy() , train_rows, dev_rows)
 train_ds, dev_ids = train_dev_split(train_data)
 data =(None, None, None, None)
 data, models = validateModels(data, models, chooser, setter )
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )<data_type_conversions>
def missing_data(data):
total = data.isnull().sum().sort_values(ascending=False)
 percent =(data.isnull().sum() /data.isnull().count() ).sort_values(ascending=False)
 md = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 return md
missing_train_data = missing_data(pd.concat([df, test], sort=True, axis=0))
missing_train_data[missing_train_data['Total']>0]
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts<feature_engineering>
def manual_cleaner(dataset):
def build_impute_map(data, missing, number, def_value):
m = {}
if number:
m = dict.fromkeys(set(data.select_dtypes(include=['number'])) &
set(missing[missing.Percent >.0].index.values), def_value)
 else:
m = dict.fromkeys(set(data.select_dtypes(exclude=['number'])) &
set(missing[missing.Percent >.0].index.values), def_value)
 return m
def man_impute(mappings, data):
for k, v in mappings.items() :
data[k].fillna(v, inplace=True)
 return data
def impute(data, missing):
numeric_map = build_impute_map(data, missing, True, 0)
 data = man_impute(numeric_map, data)
 categorical_map = build_impute_map(data, missing, False, 'None')
 categorical_map['MasVnrType'] = 'None'
categorical_map['Electrical'] = 'SBrkr'
categorical_map['Utilities'] = 'AllPub'
data = man_impute(categorical_map, data)
 return data
df_man = dataset[1].copy()
test_man = dataset[2].copy()
df_man = impute(df_man, missing_train_data)
 test_man = impute(test_man, missing_train_data)
 test_man.fillna(0, inplace=True)
 return(dataset[0], df_man, test_man, dataset[3], dataset[4])
train['revenue'] = train['item_price'] * train['item_cnt_day']<merge>
def automatic_cleaner(dataset):
def build_impute_list(data, number):
m = {}
if number:
m = [data.select_dtypes(include=['number'] ).columns.values]
else:
m = data.select_dtypes(exclude=['number'] ).columns.values
return m
def auto_impute(imputer, cols, train, test):
for col in cols:
train[col] = imputer.fit_transform(train[col])
 test[col] = imputer.transform(test[col])
 return(train, test)
 df_aut = dataset[1].copy()
test_aut = dataset[2].copy()
num_cols = build_impute_list(df_aut, True)
 cat_cols = build_impute_list(df_aut, False)
 num_imputer = SimpleImputer()
cat_imputer = CategoricalImputer()
df_aut, test_aut = auto_impute(cat_imputer, cat_cols, df_aut, test_aut)
 df_aut, test_aut = auto_impute(num_imputer, num_cols, df_aut, test_aut)
 test_aut.fillna(0, inplace=True)
 return(dataset[0], df_aut, test_aut, dataset[3], dataset[4] )
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts<data_type_conversions>
data = aut_data
models = aut_models
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
def aut_categorical_filler(dataset):
def transform(cols, train, test):
avg = train.copy().drop([523, 1298], axis=0)['SqFeetPrice'].mean()
transformer = {}
for col in cols:
transformer[col] = train.groupby(col)['SqFeetPrice'].median().map(lambda x:(x - avg)/ avg ).to_dict()
for col in cols:
train[col] = train[col].map(lambda cat: transformer[col].get(cat, 0))
test[col] = test[col].map(lambda cat: transformer[col].get(cat, 0))
return(train, test)
 train_data = dataset[1].copy()
test_data = dataset[2].copy()
train_data['Size'] =(train_data['GrLivArea'] / 600 + 1 ).astype(int)
 test_data['Size'] =(test_data['GrLivArea'] / 600 + 1 ).astype(int)
 train_data['SqFeetPrice'] = train_output.values / train_data['GrLivArea'].values
categorical_columns = list(train_data.select_dtypes(exclude=['number'] ).columns.values)
 categorical_columns.extend(['MoSold','YrSold','YearBuilt','YearRemodAdd'])
 train_data, test_data = transform(categorical_columns, train_data, test_data)
 train_data.drop(['SqFeetPrice'], axis=1, inplace=True)
 return(dataset[0], train_data, test_data, dataset[3], dataset[4])
 def all_selector(y, X):
return X
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts<data_type_conversions>
def mix_aut_categorical_filler(dataset):
def man_transform(dataset):
dataset["BsmtCond"] = dataset["BsmtCond"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["BsmtExposure"] = dataset["BsmtExposure"].astype(pd.api.types.CategoricalDtype(categories = ['None','Mn','Av','Gd'],ordered=True)).cat.codes
dataset["BsmtFinType1"] = dataset["BsmtFinType1"].astype(pd.api.types.CategoricalDtype(categories = ['None','Unf','LwQ','Rec','BLQ','ALQ','GLQ'],ordered=True)).cat.codes
dataset["BsmtFinType2"] = dataset["BsmtFinType2"].astype(pd.api.types.CategoricalDtype(categories = ['None','Unf','LwQ','Rec','BLQ','ALQ','GLQ'],ordered=True)).cat.codes
dataset["BsmtQual"] = dataset["BsmtQual"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["ExterCond"] = dataset["ExterCond"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["ExterQual"] = dataset["ExterQual"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["Fence"] = dataset["Fence"].astype(pd.api.types.CategoricalDtype(categories = ['None','MnWw','GdWo','MnPrv','GdPrv'],ordered=True)).cat.codes
dataset["FireplaceQu"] = dataset["FireplaceQu"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["Functional"] = dataset["Functional"].astype(pd.api.types.CategoricalDtype(categories = ['Sal','Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],ordered=True)).cat.codes
dataset["GarageCond"] = dataset["GarageCond"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["GarageFinish"] = dataset["GarageFinish"].astype(pd.api.types.CategoricalDtype(categories = ['None','Unf','RFn','Fin'],ordered=True)).cat.codes
dataset["GarageQual"] = dataset["GarageQual"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["HeatingQC"] = dataset["HeatingQC"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["KitchenQual"] = dataset["KitchenQual"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["PavedDrive"] = dataset["PavedDrive"].astype(pd.api.types.CategoricalDtype(categories = ['N','P','Y'],ordered=True)).cat.codes
dataset["PoolQC"] = dataset["PoolQC"].astype(pd.api.types.CategoricalDtype(categories = ['None','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["Utilities"] = dataset["Utilities"].astype(pd.api.types.CategoricalDtype(categories = ['ELO','NoSeWa','NoSewr','AllPub'],ordered=True)).cat.codes
def aut_transform(cols, train, test):
avg = train['SqFeetPrice'].mean()
transformer = {}
for col in cols:
transformer[col] = train.groupby(col)['SqFeetPrice'].mean().map(lambda x:(x - avg)/ avg ).to_dict()
for col in cols:
train[col] = train[col].map(lambda cat: transformer[col].get(cat, 0))
test[col] = test[col].map(lambda cat: transformer[col].get(cat, 0))
return(train, test)
 train_data = dataset[1].copy()
test_data = dataset[2].copy()
train_data['SqFeetPrice'] = train_output.values / train_data['GrLivArea'].values
man_transform(train_data)
 man_transform(test_data)
 categorical_columns = list(train_data.select_dtypes(exclude=['number'] ).columns.values)
 categorical_columns.extend(['MoSold','YrSold','YearBuilt','YearRemodAdd'])
 train_data, test_data = aut_transform(categorical_columns, train_data, test_data)
 train_data.drop(['SqFeetPrice'], axis=1, inplace=True)
 return(dataset[0], train_data, test_data, dataset[3], dataset[4])
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts<merge>
def mix_one_categorical_filler(dataset):
def man_transform(dataset):
dataset["BsmtCond"] = dataset["BsmtCond"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["BsmtExposure"] = dataset["BsmtExposure"].astype(pd.api.types.CategoricalDtype(categories = ['None','Mn','Av','Gd'],ordered=True)).cat.codes
dataset["BsmtFinType1"] = dataset["BsmtFinType1"].astype(pd.api.types.CategoricalDtype(categories = ['None','Unf','LwQ','Rec','BLQ','ALQ','GLQ'],ordered=True)).cat.codes
dataset["BsmtFinType2"] = dataset["BsmtFinType2"].astype(pd.api.types.CategoricalDtype(categories = ['None','Unf','LwQ','Rec','BLQ','ALQ','GLQ'],ordered=True)).cat.codes
dataset["BsmtQual"] = dataset["BsmtQual"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["ExterCond"] = dataset["ExterCond"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["ExterQual"] = dataset["ExterQual"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["Fence"] = dataset["Fence"].astype(pd.api.types.CategoricalDtype(categories = ['None','MnWw','GdWo','MnPrv','GdPrv'],ordered=True)).cat.codes
dataset["FireplaceQu"] = dataset["FireplaceQu"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["Functional"] = dataset["Functional"].astype(pd.api.types.CategoricalDtype(categories = ['Sal','Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],ordered=True)).cat.codes
dataset["GarageCond"] = dataset["GarageCond"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["GarageFinish"] = dataset["GarageFinish"].astype(pd.api.types.CategoricalDtype(categories = ['None','Unf','RFn','Fin'],ordered=True)).cat.codes
dataset["GarageQual"] = dataset["GarageQual"].astype(pd.api.types.CategoricalDtype(categories = ['None','Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["HeatingQC"] = dataset["HeatingQC"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["KitchenQual"] = dataset["KitchenQual"].astype(pd.api.types.CategoricalDtype(categories = ['Po','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["PavedDrive"] = dataset["PavedDrive"].astype(pd.api.types.CategoricalDtype(categories = ['N','P','Y'],ordered=True)).cat.codes
dataset["PoolQC"] = dataset["PoolQC"].astype(pd.api.types.CategoricalDtype(categories = ['None','Fa','TA','Gd','Ex'],ordered=True)).cat.codes
dataset["Utilities"] = dataset["Utilities"].astype(pd.api.types.CategoricalDtype(categories = ['ELO','NoSeWa','NoSewr','AllPub'],ordered=True)).cat.codes
def one_transform(cols, train, test):
union = pd.get_dummies(pd.concat([train, test], sort=False), columns=cols)
 l_train = len(train)
 train, test = union[:l_train], union[l_train:]
return(train, test)
 train_data = dataset[1].copy()
test_data = dataset[2].copy()
man_transform(train_data)
 man_transform(test_data)
 categorical_columns = list(train_data.select_dtypes(exclude=['number'] ).columns.values)
 categorical_columns.extend(['MoSold','YrSold','YearBuilt','YearRemodAdd'])
 train_data, test_data = one_transform(categorical_columns, train_data, test_data)
 return(dataset[0], train_data, test_data, dataset[3], dataset[4] )
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<merge>
def one_categorical_filler(dataset):
def transform(cols, train, test):
union = pd.get_dummies(pd.concat([train, test], sort=False), columns=cols)
 l_train = len(train)
 train, test = union[:l_train], union[l_train:]
return(train, test)
 train_data = dataset[1].copy()
test_data = dataset[2].copy()
categorical_columns = list(train_data.select_dtypes(exclude=['number'] ).columns.values)
 categorical_columns.extend(['MoSold','YrSold','YearBuilt','YearRemodAdd'])
 train_data, test_data = transform(categorical_columns, train_data, test_data)
 return(dataset[0], train_data, test_data, dataset[3], dataset[4] )
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
print('Automatically imputed dataset')
 aut_data, aut_models = validateModels(data, models, all_selector, aut_categorical_filler)
 print('Mix: manually and automatically imputed dataset')
 mix_aut_data, mix_aut_models = validateModels(data, models, all_selector, mix_aut_categorical_filler)
 print('Mix: manually imputed and one hot encoded dataset')
 mix_one_data, mix_one_models = validateModels(data, models, all_selector, mix_one_categorical_filler)
 print('One hot encoded dataset')
 one_data, one_models = validateModels(data, models, all_selector, one_categorical_filler )
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
data, models = aut_data, aut_models
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def standarizer(dataset):
train_data = dataset[1].copy()
test_data = dataset[2].copy()
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 X_scaler = RobustScaler()
train_data = pd.DataFrame(X_scaler.fit_transform(train_data), columns=train_data.columns.values)
 test_data = pd.DataFrame(X_scaler.transform(test_data), columns=test_data.columns.values)
 return(dataset[0], train_data, test_data, dataset[3], dataset[4])
 new_data, new_models = validateModels(data, models, all_selector, standarizer )
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
data, models = new_data, new_models
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
correlation_matrix = train_data.corr()
correlation_pairs = correlation_matrix.unstack()
correlation_pairs[(correlation_pairs < 1.0)&(correlation_pairs >.8)].sort_values(kind="quicksort", ascending=False )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
train_data.corrwith(train_output ).sort_values(ascending=False ).head(50 )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def corr_features_remover(dataset):
cols_to_drop = ['Exterior1st', 'SaleCondition', 'GarageCars', 'GarageCond', 'TotRmsAbvGrd', 'GarageYrBlt', 'TotalBsmtSF']
return(dataset[0], dataset[1].drop(cols_to_drop, axis=1), dataset[2].drop(cols_to_drop, axis=1), dataset[3], dataset[4])
 man_data, man_models = validateModels(data, models, all_selector, corr_features_remover )
ts = time.time()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
data, models = man_data, man_models
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def normalizer(dataset):
output_data = dataset[0].copy()
train_data = dataset[1].copy()
test_data = dataset[2].copy()
skew = train_data.skew().abs()
features = skew[skew >.7].index.values
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 feature_transformer = PowerTransformer(method='yeo-johnson')
 train_data = pd.DataFrame(feature_transformer.fit_transform(train_data), columns=train_data.columns.values)
 test_data = pd.DataFrame(feature_transformer.transform(test_data), columns=test_data.columns.values)
 output_data = pd.Series(np.log1p(output_data.values.reshape(-1,1)).reshape(-1))
return(output_data, train_data, test_data, dataset[3], dataset[4])
 new_data, new_models = validateModels(data, models, all_selector, normalizer )
ts = time.time()
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
data, models = new_data, new_models
ts = time.time()
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<data_type_conversions>
def outlier_remover(dataset):
output_data = dataset[0].copy()
train_data = dataset[1].copy()
outliers = []
outliers.extend(train_data[(train_output < 12.5)&(train_data.GrLivArea > 3)].index.values)
 outliers.extend(train_data[(train_output < 12.5)&(train_data.GarageArea > 3)].index.values)
 outliers.extend(train_data[(train_output < 11)&(train_data.OpenPorchSF > 0.5)].index.values)
 outliers.extend(train_data[(train_output < 12.5)&(train_data['1stFlrSF'] > 4)].index.values)
 train_data = train_data.drop(list(set(outliers)) , axis=0)
 output_data = output_data.drop(list(set(outliers)) , axis=0)
 train_rows, dev_rows = train_dev_split(train_data)
 return(output_data, train_data, dataset[2], train_rows, dev_rows)
 new_data, new_models = validateModels(data, models, all_selector, outlier_remover)
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts<merge>
data, models = new_data, new_models
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts<feature_engineering>
def tune(data, models, params):
p = {}
results = pd.DataFrame(columns=['model', 'result_index', 'mean_test_score', 'std_test_score', 'mean_train_score', 'std_train_score'])
 for(k, v)in params.items() :
if models[k] is not None:
model, selector = models[k] 
(output_data, train_data, test_data, _, _)= data
search = GridSearchCV(model, v, scoring='r2', cv=5, iid=False)
 with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 search.fit(selector(output_data, train_data), output_data)
 size = len(search.cv_results_['mean_test_score'])
 p[k] = search.cv_results_['params']
d_f = pd.DataFrame({'model' : [k] * size,
'result_index' : range(size),
'mean_test_score' : search.cv_results_['mean_test_score'],
'std_test_score' : search.cv_results_['std_test_score'],
'mean_train_score' : search.cv_results_['mean_train_score'],
'std_train_score' : search.cv_results_['std_train_score']})
 d_f.head(4)
 results = pd.concat([results, d_f], axis=0)
 return p, results
def applyParams(models, params):
def proces_params(ps):
result = {}
for k, v in ps.items() :
result[k] = v[0]
return result
for(k, v)in params.items() :
if models[k] is not None:
model, _ = models[k]
ps = proces_params(v)
 model.set_params(**ps )
matrix['month'] = matrix['date_block_num'] % 12<categorify>
lr_params = {'fit_intercept': [True, False],
'n_jobs': [-1],
'normalize': [True, False]}
fitting_params = {
'linear_regression' : lr_params
}
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )<data_type_conversions>
params, results = tune(data, models, fitting_params )
ts = time.time()
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts<data_type_conversions>
results.sort_values(by='mean_test_score', ascending=False ).head(32 )
<feature_engineering><EOS> 
applyParams(models, fitting_params )