Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<prepare_x_and_y>
%matplotlib inline
X_train= resDF[resDF['date_block_num']<33].drop(['target'],axis=1)
 X_test= resDF[resDF['date_block_num']==34].drop(['target'],axis=1)
 X_val= resDF[resDF['date_block_num']==33].drop(['target'],axis=1)
 Y_train= resDF[resDF['date_block_num']<33]['target']
Y_val= resDF[resDF['date_block_num']==33]['target']
xgbModel= model;lrModel= lr;<predict_on_test>
df_train = pd.read_csv('.. /input/train.csv',index_col='Id')
 df_test = pd.read_csv('.. /input/test.csv',index_col='Id')
 id_train = df_train.index
id_test = df_test.index
df_all = pd.concat([df_train,df_test], sort=True)
 df_all.head(5 )
meta_xg_train= xgbModel.predict(X_train)
 meta_lr_train= lrModel.predict(X_train)
 meta_xg_val= xgbModel.predict(X_val)
 meta_lr_val= lrModel.predict(X_val)
 meta_xg_test= xgbModel.predict(X_test)
 meta_lr_test= lrModel.predict(X_test)
 metaTrain= np.c_[meta_xg_train,meta_lr_train]
print(metaTrain.shape,Y_train.shape)
 metaVal= np.c_[meta_xg_val,meta_lr_val]
print(metaVal.shape,Y_val.shape)
 metaTest= np.c_[meta_xg_test,meta_lr_test]
print(metaTest.shape )<compute_test_metric>
cols_with_na = df_all.isnull().sum()
cols_with_na = cols_with_na[cols_with_na>0]
print(cols_with_na.sort_values(ascending=False))
lr= LinearRegression(n_jobs=-1)
 lr.fit(metaTrain,Y_train.values)
 metaVal= np.c_[meta_xg_val,meta_lr_val]
Y_train_pred= lr.predict(metaTrain)
 Y_val_pred= lr.predict(metaVal)
 print(np.sqrt(mean_squared_error(Y_val_pred,Y_val.values)))
 print(np.sqrt(mean_squared_error(Y_train_pred,Y_train.values)) )<save_to_csv>
cols_fillna = ['PoolQC','MiscFeature','Alley','Fence','MasVnrType','FireplaceQu',
'GarageQual','GarageCond','GarageFinish','GarageType',
'BsmtExposure','BsmtCond','BsmtQual','BsmtFinType1','BsmtFinType2']
for col in cols_fillna:
df_all[col].fillna('None',inplace=True)
 df_all.loc[df_all.GarageYrBlt.isnull() ,'GarageYrBlt'] = df_all.loc[df_all.GarageYrBlt.isnull() ,'YearBuilt']
df_all.MasVnrArea.fillna(0,inplace=True)
 df_all.BsmtFullBath.fillna(0,inplace=True)
 df_all.BsmtHalfBath.fillna(0,inplace=True)
 df_all.BsmtFinSF1.fillna(0,inplace=True)
 df_all.BsmtFinSF2.fillna(0,inplace=True)
 df_all.BsmtUnfSF.fillna(0,inplace=True)
 df_all.TotalBsmtSF.fillna(0,inplace=True)
 df_all.GarageArea.fillna(0,inplace=True)
 df_all.GarageCars.fillna(0,inplace=True)
 df_all[cols_fillna].head(5 )
drop_cols= [ 'target']
testDF = resDF[ resDF['date_block_num']==34]
testDF['target']= lr.predict(metaTest)
 submission = pd.DataFrame({
"ID": testDF['ID'],
"item_cnt_month": testDF['target'].clip(0,20)
 })
 submission['ID']= submission['ID'].astype(np.int32)
 submission.to_csv('submission.csv', index=False )<load_from_csv>
df_frontage = pd.get_dummies(df_all.drop('SalePrice',axis=1))
for col in df_frontage.drop('LotFrontage',axis=1 ).columns:
df_frontage[col] = scale_minmax(df_frontage[col])
 lf_train = df_frontage.dropna()
lf_train_y = lf_train.LotFrontage
lf_train_X = lf_train.drop('LotFrontage',axis=1)
 lr = Ridge()
lr.fit(lf_train_X, lf_train_y)
 lr_coefs = pd.Series(lr.coef_,index=lf_train_X.columns)
 print('----------------')
 print('Intercept:',lr.intercept_)
 print('----------------')
 print(lr_coefs.sort_values(ascending=False ).head(10))
print('----------------')
 print(lr_coefs.sort_values(ascending=False ).tail(10))
print('----------------')
 print('R2:',lr.score(lf_train_X,lf_train_y))
print('----------------')
 lf_pred_y = lr.predict(lf_train_X)
 plt.figure(figsize=(18,5))
plt.subplot(1,3,1)
 plt.plot(lf_train_y,lf_pred_y,'.')
 plt.xlabel('y')
 plt.ylabel('y_pred');
plt.title('Prediction vs.True Value')
 plt.subplot(1,3,2)
 plt.plot(lf_train_y, lf_train_y-lf_pred_y,'.')
 plt.xlabel('y')
 plt.ylabel('y - y_pred');
plt.title('Residual vs.True Value')
 nan_frontage = df_all.LotFrontage.isnull()
X = df_frontage[nan_frontage].drop('LotFrontage',axis=1)
 y = lr.predict(X)
 plt.subplot(1,3,3)
 plt.hist(y,bins=20)
 plt.title('Predicted Lot Frontages for NaN Values')
 df_all.loc[nan_frontage,'LotFrontage'] = y
train = pd.read_csv('.. /input/sales_train.csv')
 items = pd.read_csv('.. /input/items.csv')
 categories = pd.read_csv('.. /input/item_categories.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 test = pd.read_csv('.. /input/test.csv')
 submission = pd.read_csv('.. /input/sample_submission.csv' )<count_values>
cols_with_na = df_all.drop('SalePrice',axis=1 ).isnull().sum()
cols_with_na = cols_with_na[cols_with_na>0]
print(cols_with_na.sort_values(ascending=False))
rows_with_na = df_all.drop('SalePrice',axis=1 ).isnull().sum(axis=1)
 rows_with_na = rows_with_na[rows_with_na>0]
print(rows_with_na.sort_values(ascending=False))
for col in cols_with_na.index:
df_all[col].fillna(df_all[col].mode() [0], inplace=True )
train['item_id'].value_counts(ascending=False)[:5]<filter>
bsmt_fin_cols = ['BsmtGLQ','BsmtALQ','BsmtBLQ',
'BsmtRec','BsmtLwQ']
for col in bsmt_fin_cols:
df_all[col+'SF'] = 0
for row in df_all.index:
fin1 = df_all.loc[row,'BsmtFinType1']
if(fin1!='None')and(fin1!='Unf'):
df_all.loc[row,'Bsmt'+fin1+'SF'] += df_all.loc[row,'BsmtFinSF1']
fin2 = df_all.loc[row,'BsmtFinType2']
if(fin2!='None')and(fin2!='Unf'):
df_all.loc[row,'Bsmt'+fin2+'SF'] += df_all.loc[row,'BsmtFinSF2']
df_all.drop(['BsmtFinType1','BsmtFinSF1','BsmtFinType2','BsmtFinSF2'], axis=1, inplace=True)
 bsmt_fin_cols.append('BsmtUnf')
 for col in bsmt_fin_cols:
df_all[col+'Frac'] = df_all[col+'SF']/df_all['TotalBsmtSF']
df_all[col+'Frac'].fillna(0,inplace=True)
 plt.figure(figsize=(15,4))
plt.subplot(1,2,1)
 sns.regplot(df_all.loc[(df_all['BsmtUnfFrac']>0)&(df_all['BsmtUnfFrac']<1), 'BsmtUnfSF'],
df_all.loc[(df_all['BsmtUnfFrac']>0)&(df_all['BsmtUnfFrac']<1), 'SalePrice'],
scatter_kws={'marker':'.','s':3,'alpha':0.5,'color':'r'},
line_kws={'color':'k'})
 plt.xlabel('BsmtUnf SF')
 plt.title('Before')
 plt.subplot(1,2,2)
 sns.regplot(df_all.loc[(df_all['BsmtUnfFrac']>0)&(df_all['BsmtUnfFrac']<1), 'BsmtUnfFrac'],
df_all.loc[(df_all['BsmtUnfFrac']>0)&(df_all['BsmtUnfFrac']<1), 'SalePrice'],
scatter_kws={'marker':'.','s':3,'alpha':0.5,'color':'b'},
line_kws={'color':'k'})
 plt.xlabel('BsmtUnf Fraction')
 plt.title('After')
 df_all.loc[:,df_all.columns.str.contains('Bsmt')].head(5 )
items.loc[items['item_id']==20949]<filter>
df_all['LowQualFinFrac'] = df_all['LowQualFinSF']/df_all['GrLivArea']
df_all['1stFlrFrac'] = df_all['1stFlrSF']/df_all['GrLivArea']
df_all['2ndFlrFrac'] = df_all['2ndFlrSF']/df_all['GrLivArea']
df_all['TotalAreaSF'] = df_all['GrLivArea']+df_all['TotalBsmtSF']+df_all['GarageArea']+df_all['EnclosedPorch']+df_all['ScreenPorch']
df_all['LivingAreaSF'] = df_all['1stFlrSF'] + df_all['2ndFlrSF'] + df_all['BsmtGLQSF'] + df_all['BsmtALQSF'] + df_all['BsmtBLQSF']
df_all['StorageAreaSF'] = df_all['LowQualFinSF'] + df_all['BsmtRecSF'] + df_all['BsmtLwQSF'] + df_all['BsmtUnfSF'] + df_all['GarageArea']
categories.loc[categories['item_category_id']==71]<sort_values>
cols_ExGd = ['ExterQual','ExterCond','BsmtQual','BsmtCond',
'HeatingQC','KitchenQual','FireplaceQu','GarageQual',
'GarageCond','PoolQC']
dict_ExGd = {'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'None':0}
for col in cols_ExGd:
df_all[col].replace(dict_ExGd, inplace=True)
 display(df_all[cols_ExGd].head(5))
df_all['BsmtExposure'].replace({'Gd':4,'Av':3,'Mn':2,'No':1,'None':0}, inplace=True)
 df_all['CentralAir'].replace({'Y':1,'N':0}, inplace=True)
 df_all['Functional'].replace({'Typ':7,'Min1':6,'Min2':5,'Mod':4,'Maj1':3,'Maj2':2,'Sev':1,'Sal':0}, inplace=True)
 df_all['GarageFinish'].replace({'Fin':3,'RFn':2,'Unf':1,'None':0}, inplace=True)
 df_all['LotShape'].replace({'Reg':3,'IR1':2,'IR2':1,'IR3':0}, inplace=True)
 df_all['Utilities'].replace({'AllPub':3,'NoSewr':2,'NoSeWa':1,'ELO':0}, inplace=True)
 df_all['LandSlope'].replace({'Gtl':2,'Mod':1,'Sev':0}, inplace=True )
train['item_cnt_day'].sort_values(ascending=False)[:5]<filter>
frac_zeros =(( df_all==0 ).sum() /len(df_all))
n_unique = df_all.nunique()
xs_zeros = frac_zeros - 1/n_unique
zero_cols = pd.DataFrame({'frac_zeros':frac_zeros,'n_unique':n_unique,'xs_zeros':xs_zeros})
 zero_cols = zero_cols[zero_cols.frac_zeros>0]
zero_cols.sort_values(by='xs_zeros',ascending=False,inplace=True)
 display(zero_cols[(zero_cols.xs_zeros>0)])
 df_all['HasPool'] =(df_all['PoolQC']>0 ).astype(int)
 df_all['Has3SsnPorch'] =(df_all['3SsnPorch']>0 ).astype(int)
 df_all.drop(['PoolQC','PoolArea','3SsnPorch'],axis=1,inplace=True)
 df_all['BsmtFullBath'] = df_all['BsmtFullBath'] + 0.5*df_all['BsmtHalfBath']
df_all['FullBath'] = df_all['FullBath'] + 0.5*df_all['HalfBath']
df_all.drop(['BsmtHalfBath','HalfBath'],axis=1,inplace=True)
 dummy_cols = ['LowQualFinSF','2ndFlrSF',
'MiscVal','ScreenPorch','WoodDeckSF','OpenPorchSF',
'EnclosedPorch','MasVnrArea','GarageArea','Fireplaces',
'BsmtGLQSF','BsmtALQSF','BsmtBLQSF','BsmtRecSF',
'BsmtLwQSF','BsmtUnfSF','TotalBsmtSF']
for col in dummy_cols:
df_all['Has'+col] =(df_all[col]>0 ).astype(int )
train[train['item_cnt_day'] == 2169]<filter>
dtypes = df_all.dtypes
cols_numeric = dtypes[dtypes != object].index.tolist()
cols_numeric.remove('MSSubClass')
 col_nunique = dict()
for col in cols_numeric:
col_nunique[col] = df_all[col].nunique()
col_nunique = pd.Series(col_nunique)
 cols_discrete = col_nunique[col_nunique<13].index.tolist()
cols_continuous = col_nunique[col_nunique>=13].index.tolist()
print(len(cols_numeric),'numeric columns, of which',
len(cols_continuous),'are continuous and',
len(cols_discrete),'are discrete.' )
items[items['item_id'] == 11373]<filter>
cols_categ = dtypes[~dtypes.index.isin(cols_numeric)].index.tolist()
for col in cols_categ:
df_all[col] = df_all[col].astype('category')
 print(len(cols_categ),'categorical columns.' )
train = train[train['item_cnt_day'] < 2000]<sort_values>
def anova(group,value):
data = df_all[[group,value]]
data = data[~(data[group].isnull() | data[value].isnull())]
tot_groups = data[group].nunique()
len_data = len(data)
 mean_data = data[value].mean()
df_betwn = tot_groups - 1
df_within = len_data - tot_groups
n_in_group = data.groupby(group)[value].count()
mean_group = data.groupby(group)[value].mean()
betwn_var = n_in_group*(( mean_group - mean_data)**2)
 betwn_var = float(betwn_var.sum())/df_betwn
within_var = 0
for grp in data[group].unique() :
samples = data.loc[data[group]==grp, value]
within_var +=(( samples-mean_group[grp])**2 ).sum()
within_var = float(within_var)/df_within
F = betwn_var/within_var
p = stats.f.sf(F, df_betwn, df_within)
 return p
train['item_price'].sort_values(ascending=False)[:5]<filter>
cols_notransform = ['2ndFlrSF','1stFlrFrac','2ndFlrFrac','StorageAreaSF',
'EnclosedPorch','LowQualFinSF','MasVnrArea',
'MiscVal','ScreenPorch','OpenPorchSF','WoodDeckSF','SalePrice',
'BsmtGLQSF','BsmtALQSF','BsmtBLQSF','BsmtRecSF','BsmtLwQSF','BsmtUnfSF',
'BsmtGLQFrac','BsmtALQFrac','BsmtBLQFrac','BsmtRecFrac','BsmtLwQFrac','BsmtUnfFrac']
cols_transform = [col for col in cols_continuous if col not in cols_notransform]
print('Transforming',len(cols_transform),'columns:',cols_transform)
 for col in cols_transform:
df_all.loc[:,col], _ = stats.boxcox(df_all.loc[:,col]+1)
 df_all.loc[:,col] = scale_minmax(df_all.loc[:,col] )
train[train['item_price'] == 307980]<filter>
model_cols = df_all.columns
df_model = pd.get_dummies(df_all[model_cols])
 if('Condition1' in model_cols)and('Condition2' in model_cols):
cond_suffix = ['Artery','Feedr','Norm','PosA','PosN','RRAe','RRAn','RRNn']
for suffix in cond_suffix:
col_cond1 = 'Condition1_'+suffix
col_cond2 = 'Condition2_'+suffix
df_model[col_cond1] = df_model[col_cond1] | df_model[col_cond2]
df_model.drop(col_cond2,axis=1,inplace=True)
 if('Exterior1st' in model_cols)and('Exterior2nd' in model_cols):
df_model.rename(columns={'Exterior2nd_Wd Shng':'Exterior2nd_WdShing',
'Exterior2nd_Brk Cmn':'Exterior2nd_BrkComm',
'Exterior2nd_CmentBd':'Exterior2nd_CemntBd'},inplace=True)
 ext_suffix = ['AsphShn','BrkComm','BrkFace','CBlock','CemntBd',
'HdBoard','ImStucc','MetalSd','Plywood','Stone',
'Stucco','VinylSd','Wd Sdng','WdShing','AsbShng']
for suffix in ext_suffix:
col_cond1 = 'Exterior1st_'+suffix
col_cond2 = 'Exterior2nd_'+suffix
df_model[col_cond1] = df_model[col_cond1] | df_model[col_cond2]
df_model.drop(col_cond2,axis=1,inplace=True)
 display(df_model.head() )
items[items['item_id'] == 6066]<filter>
def get_training_data() :
df_train = df_model.loc[id_train]
y = df_train.SalePrice
X = df_train.drop('SalePrice',axis=1)
 return X, y
def get_test_data() :
return df_model.loc[id_test].drop('SalePrice',axis=1 )
train[train['item_id'] == 6066]<filter>
def rmse(y_true, y_pred):
diff = y_pred - y_true
sum_sq = sum(diff**2)
 n = len(y_pred)
 return np.sqrt(sum_sq/n)
 rmse_scorer = make_scorer(rmse, greater_is_better=False )
train = train[train['item_price'] < 300000]<sort_values>
def find_outliers(model, X, y, sigma=3):
try:
y_pred = pd.Series(model.predict(X), index=y.index)
 except:
model.fit(X,y)
 y_pred = pd.Series(model.predict(X), index=y.index)
 resid = y - y_pred
mean_resid = resid.mean()
std_resid = resid.std()
z =(resid - mean_resid)/std_resid
outliers = z[abs(z)>sigma].index
print('R2=',model.score(X,y))
print('rmse=',rmse(y, y_pred))
print('---------------------------------------')
 print('mean of residuals:',mean_resid)
 print('std of residuals:',std_resid)
 print('---------------------------------------')
 print(len(outliers),'outliers:')
 print(outliers.tolist())
 plt.figure(figsize=(15,5))
ax_131 = plt.subplot(1,3,1)
 plt.plot(y,y_pred,'.')
 plt.plot(y.loc[outliers],y_pred.loc[outliers],'ro')
 plt.legend(['Accepted','Outlier'])
 plt.xlabel('y')
 plt.ylabel('y_pred');
ax_132=plt.subplot(1,3,2)
 plt.plot(y,y-y_pred,'.')
 plt.plot(y.loc[outliers],y.loc[outliers]-y_pred.loc[outliers],'ro')
 plt.legend(['Accepted','Outlier'])
 plt.xlabel('y')
 plt.ylabel('y - y_pred');
ax_133=plt.subplot(1,3,3)
 z.plot.hist(bins=50,ax=ax_133)
 z.loc[outliers].plot.hist(color='r',bins=50,ax=ax_133)
 plt.legend(['Accepted','Outlier'])
 plt.xlabel('z')
 plt.savefig('outliers.png')
 return outliers
train['item_price'].sort_values() [:5]<filter>
X, y = get_training_data()
outliers = find_outliers(Ridge() , X, y)
 df_model = df_model.drop(outliers)
 id_train = id_train.drop(outliers )
train[train['item_price'] == -1]<feature_engineering>
def train_model(model, param_grid=[], X=[], y=[],
splits=5, repeats=5):
if len(y)==0:
X,y = get_training_data()
rkfold = RepeatedKFold(n_splits=splits, n_repeats=repeats)
 if len(param_grid)>0:
gsearch = GridSearchCV(model, param_grid, cv=rkfold,
scoring=rmse_scorer,
verbose=1, return_train_score=True)
 gsearch.fit(X,y)
 model = gsearch.best_estimator_
best_idx = gsearch.best_index_
grid_results = pd.DataFrame(gsearch.cv_results_)
 cv_mean = abs(grid_results.loc[best_idx,'mean_test_score'])
 cv_std = grid_results.loc[best_idx,'std_test_score']
else:
grid_results = []
cv_results = cross_val_score(model, X, y, scoring=rmse_scorer, cv=rkfold)
 cv_mean = abs(np.mean(cv_results))
cv_std = np.std(cv_results)
 cv_score = pd.Series({'mean':cv_mean,'std':cv_std})
 y_pred = model.predict(X)
 print('----------------------')
 print(model)
 print('----------------------')
 print('score=',model.score(X,y))
print('rmse=',rmse(y, y_pred))
print('cross_val: mean=',cv_mean,', std=',cv_std)
 y_pred = pd.Series(y_pred,index=y.index)
 resid = y - y_pred
mean_resid = resid.mean()
std_resid = resid.std()
z =(resid - mean_resid)/std_resid
n_outliers = sum(abs(z)>3)
 plt.figure(figsize=(15,5))
ax_131 = plt.subplot(1,3,1)
 plt.plot(y,y_pred,'.')
 plt.xlabel('y')
 plt.ylabel('y_pred');
plt.title('corr = {:.3f}'.format(np.corrcoef(y,y_pred)[0][1]))
ax_132=plt.subplot(1,3,2)
 plt.plot(y,y-y_pred,'.')
 plt.xlabel('y')
 plt.ylabel('y - y_pred');
plt.title('std resid = {:.3f}'.format(std_resid))
ax_133=plt.subplot(1,3,3)
 z.plot.hist(bins=50,ax=ax_133)
 plt.xlabel('z')
 plt.title('{:.0f} samples with z>3'.format(n_outliers))
return model, cv_score, grid_results
price_correction = train[(train['shop_id'] == 32)&(train['item_id'] == 2973)&(train['date_block_num'] == 4)&(train['item_price'] > 0)].item_price.median()
train.loc[train['item_price'] < 0, 'item_price'] = price_correction<count_unique_values>
opt_models = dict()
score_models = pd.DataFrame(columns=['mean','std'])
 splits=5
repeats=5
shops_train = train['shop_id'].nunique()
shops_test = test['shop_id'].nunique()
print('Shops in Training Set: ', shops_train)
 print('Shops in Test Set: ', shops_test )<feature_engineering>
model ='ElasticNet'
opt_models[model] = ElasticNet()
param_grid = {'alpha': np.arange(1e-4,1e-3,1e-4),
'l1_ratio': np.arange(0.1,1.0,0.1),
'max_iter':[100000]}
opt_models[model], cv_score, grid_results = train_model(opt_models[model], param_grid=param_grid,
splits=splits, repeats=repeats)
 cv_score.name = model
score_models = score_models.append(cv_score )
train.loc[train['shop_id'] == 0, 'shop_id'] = 57
test.loc[test['shop_id'] == 0, 'shop_id'] = 57
train.loc[train['shop_id'] == 1, 'shop_id'] = 58
test.loc[test['shop_id'] == 1, 'shop_id'] = 58
train.loc[train['shop_id'] == 10, 'shop_id'] = 11
test.loc[test['shop_id'] == 10, 'shop_id'] = 11<feature_engineering>
model ='SVR'
opt_models[model] = SVR()
param_grid = {'C':np.arange(1,21,2),
'kernel':['poly','rbf','sigmoid'],
'gamma':['auto']}
opt_models[model], cv_score, grid_results = train_model(opt_models[model], param_grid=param_grid,
splits=splits, repeats=1)
 cv_score.name = model
score_models = score_models.append(cv_score )
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda row: row[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'<categorify>
model = 'RandomForest'
opt_models[model] = RandomForestRegressor()
param_grid = {'n_estimators':[100,150,200],
'max_features':[25,50,75],
'min_samples_split':[2,4,6]}
opt_models[model], cv_score, grid_results = train_model(opt_models[model], param_grid=param_grid,
splits=5, repeats=1)
 cv_score.name = model
score_models = score_models.append(cv_score )
le = preprocessing.LabelEncoder()
le.fit_transform(shops['city'] )<categorify>
model = 'GradientBoosting'
opt_models[model] = GradientBoostingRegressor()
param_grid = {'n_estimators':[150,250,350],
'max_depth':[1,2,3],
'min_samples_split':[5,6,7]}
opt_models[model], cv_score, grid_results = train_model(opt_models[model], param_grid=param_grid,
splits=splits, repeats=1)
 cv_score.name = model
score_models = score_models.append(cv_score )
shops['city_label'] = le.fit_transform(shops['city'])
 shops.drop(['shop_name', 'city'], axis = 1, inplace = True)
 shops.head()<count_unique_values>
model = 'XGB'
opt_models[model] = XGBRegressor()
param_grid = {'n_estimators':[100,200,300,400,500],
'max_depth':[1,2,3],
}
opt_models[model], cv_score,grid_results = train_model(opt_models[model], param_grid=param_grid,
splits=splits, repeats=1)
 cv_score.name = model
score_models = score_models.append(cv_score )
items_train = train['item_id'].nunique()
items_test = test['item_id'].nunique()
print('Items in Training Set: ', items_train)
 print('Items in Test Set: ', items_test )<concatenate>
X,y=get_training_data()
select_cols = [col for col in X.columns if X[col].nunique() >2 and en_coefs[col]!=0]
X = X[select_cols]
poly = PolynomialFeatures(interaction_only=True)
 X_poly = poly.fit_transform(X,y)
 X_poly = pd.DataFrame(X_poly,index=y.index)
 powers = pd.DataFrame(poly.powers_,columns=X.columns)
 s = np.where(powers>0, pd.Series(X.columns)+', ', '')
 poly_terms = pd.Series([''.join(x ).strip() for x in s])
 alph_range = np.arange(5e-5,2e-4,2e-5)
 param_grid = {'alpha': np.arange(1e-4,1e-3,1e-4),
'max_iter':[100000]}
model,_,_ = train_model(Lasso() , X=X_poly,y=y,param_grid=param_grid,
splits=splits, repeats=1)
 poly_coefs = pd.Series(model.coef_)
 print('------------------------')
 print(sum(poly_coefs==0),'zero coefficients')
 print(sum(poly_coefs!=0),'non-zero coefficients')
 print(len(poly_coefs[(powers.sum(axis=1)==2)&(poly_coefs>0)]),'non-zero interaction terms.')
 print('------------------------')
 print('Features with largest coefficients:')
 print('------------------------')
 print(poly_terms[poly_coefs.abs().sort_values(ascending=False ).index[:30]])
 print('------------------------' )
len(set(items_test_list ).difference(items_train_list))<filter>
poly_coef_nterms = powers.loc[poly_coefs.abs().sort_values(ascending=False ).index].sum(axis=1)
 n_ints = 20
top_n_int_idx = poly_coef_nterms[poly_coef_nterms==2].head(n_ints ).index
for idx in top_n_int_idx:
int_terms = powers.loc[idx]
int_terms = int_terms[int_terms==1].index
term1 = int_terms[0]
term2 = int_terms[1]
df_model[term1+'_x_'+term2] = scale_minmax(df_model[term1]*df_model[term2])
 display(df_model[df_model.columns[df_model.columns.str.contains('_x_')]].head(5))
categories.loc[~categories['item_category_id'].isin(categories_in_test)].T<categorify>
en_int = ElasticNet()
alph_range = np.arange(5e-5,5e-4,2e-5)
 param_grid = {'alpha': np.arange(1e-4,1e-3,1e-4),
'l1_ratio': np.arange(0.1,1.0,0.1),
'max_iter':[100000]}
en_int, cv_score, grid_results = train_model(en_int, param_grid=param_grid,
splits=splits, repeats=repeats)
 X,y=get_training_data()
en_int_coef = pd.Series(en_int.coef_,index=X.columns)
 print('------------------------')
 print(sum(en_int_coef==0),'zero coefficients')
 print(sum(en_int_coef!=0),'non-zero coefficients')
 print('--------------------------')
 print('Interaction Coefficients:')
 print('--------------------------')
 print(en_int_coef[en_int_coef.index.str.contains('_x_')].sort_values(ascending=False))
print('---------------------------------------')
 print('Top 10 contributers to increased price:')
 print('---------------------------------------')
 print(en_int_coef.sort_values(ascending=False ).head(10))
print('---------------------------------------')
 print('Top 5 contributers to decreased price:')
 print('---------------------------------------')
 print(en_int_coef.sort_values(ascending=True ).head(5))
print('---------------------------------------' )
le = preprocessing.LabelEncoder()
main_categories = categories['item_category_name'].str.split('-')
 categories['main_category_id'] = main_categories.map(lambda row: row[0].strip())
 categories['main_category_id'] = le.fit_transform(categories['main_category_id'])
 categories['sub_category_id'] = main_categories.map(lambda row: row[1].strip() if len(row)> 1 else row[0].strip())
 categories['sub_category_id'] = le.fit_transform(categories['sub_category_id'] )<data_type_conversions>
def predict_test(model):
X_test = get_test_data()
y_test = model.predict(X_test)
 y_test = np.exp(y_test)
 y_test = pd.DataFrame(y_test,index=X_test.index)
 y_test.columns = ['SalePrice']
return y_test
train['date'] = pd.to_datetime(train['date'], format='%d.%m.%Y' )<import_modules>
y_test = predict_test(en_int)
 y_test.to_csv('submission.csv')
 y_test.plot.hist(bins=20 )