Kaggle competition: House Prices - Advanced Regression Techniques
%%time
resDF= cartesianDF.merge(targetDF,how='left',on=['shop_id','item_id','date_block_num'] )<merge>
house_data = pd.read_csv('.. /input/train.csv')
 house_data_test = pd.read_csv('.. /input/test.csv' )
itemPrice= salesData.groupby(['item_id'],as_index=False ).agg({'item_price':'mean'})
 itemPrice.columns=['item_id','avg_price_item']
resDF['avg_price'][ resDF['avg_price'].isna() ]= resDF[ resDF['avg_price'].isna() ].merge(itemPrice,how='left',on=['item_id'])['avg_price_item'].values<categorify>
train_parent=house_data
test_parent=house_data_test
house_data = house_data.drop('Id', axis=1)
 house_data_test = house_data_test.drop('Id', axis=1 )
shops['shop_label']= shops['shop_name'].apply(lambda x: x.split(' ')[0])
 le = preprocessing.LabelEncoder()
shops['shop_label']= le.fit_transform(shops['shop_label'] )<categorify>
threshold=0.4 * len(house_data)
 df=pd.DataFrame(len(house_data)- house_data.count() ,columns=['count'])
 df.index[df['count'] > threshold]
itemCategories['cat_label']= itemCategories['item_category_name'].apply(lambda x: x.split('-')[0].strip())
 itemCategories['cat_subLabel']= itemCategories['item_category_name'].apply(lambda x: x.split('-')[1].strip() if '-' in x else x.split('-')[0].strip())
 le = preprocessing.LabelEncoder()
itemCategories['cat_label']= le.fit_transform(itemCategories['cat_label'])
 itemCategories['cat_subLabel']= le.fit_transform(itemCategories['cat_subLabel'] )<merge>
house_data = house_data.drop(['Alley', 'FireplaceQu', 'PoolQC', 'Fence', 'MiscFeature'], axis=1)
 house_data_test = house_data_test.drop(['Alley', 'FireplaceQu', 'PoolQC', 'Fence', 'MiscFeature'], axis=1 )
resDF= resDF.merge(items[['item_id','item_category_id']],on=['item_id'],how='left')
 resDF= resDF.merge(itemCategories[['cat_label','cat_subLabel','item_category_id']],how='left',on=['item_category_id'])
 resDF= resDF.merge(shops[['shop_label','shop_id']],how='left',on=['shop_id'] )<define_variables>
house_data.select_dtypes(include=np.number ).columns
for col in('MSSubClass', 'LotFrontage', 'LotArea', 'OverallQual', 'OverallCond',
'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2',
'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'LowQualFinSF',
'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath',
'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces',
'GarageYrBlt', 'GarageCars', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF',
'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal',
'MoSold', 'YrSold'):
house_data[col] = house_data[col].fillna(0)
 house_data_test[col] = house_data_test[col].fillna('0' )
meanEncodingCols =[ 
(['shop_id'],[1,2,3,6,12]), 
(['item_id'],[1,2,3,6,12]), 
(['item_category_id'],[1,2,3,6,12]), 
(['item_id','shop_id'],[1,2,3,6,12]), 
(['item_category_id','shop_id'],[1,2,3,6,12]), 
(['cat_label'],[1]), 
(['cat_label','item_id'],[1,2,3]), 
(['cat_subLabel'],[1]), 
(['shop_label'],[1])
 ]<groupby>
house_data.select_dtypes(exclude=np.number ).columns
for col in('MSZoning', 'Street', 'LotShape', 'LandContour', 'Utilities',
'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2',
'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st',
'Exterior2nd', 'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',
'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'KitchenQual',
'Functional', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond',
'PavedDrive', 'SaleType', 'SaleCondition'):
house_data[col] = house_data[col].fillna('None')
 house_data_test[col] = house_data_test[col].fillna('None' )
def generateGroupByFeatures(ipDF,groupCols,aggDict):
groupByAlias = {'item_id':'item','shop_id':'shop','item_category_id':'item_cat','date_block_num':'date',
'month':'month','target':'target','cat_label':'cat_label','cat_subLabel':'cat_subLabel',
'shop_label':'shop_label'}
aggColAlias= {'item_cnt_day':'cnt','item_price':'price','target':'target','avg_price':'price'}
grpByName= '_'.join([groupByAlias[x] for x in groupCols])+'_'
colNameList=[]
for aggCol,aggFnArr in aggDict.items() :
for f in aggFnArr:
fName = f.__name__ if hasattr(f, '__call__')else f
colNameList.append(grpByName+aggColAlias[aggCol]+"_"+fName)
 returnDF= ipDF.groupby(groupCols,as_index=False ).agg(aggDict)
 returnDF.columns= groupCols+colNameList
return returnDF<data_type_conversions>
house_data[house_data.isnull().any(axis=1)]
for c in resDF.columns:
if 'int' in str(resDF[c].dtype):
resDF[c]= resDF[c].astype('Int32')
 if resDF[c].max() <32000:
resDF[c]= resDF[c].astype('Int16')
 if resDF[c].max() <120:
resDF[c]= resDF[c].astype('Int8')
 if 'float' in str(resDF[c].dtype):
resDF[c]= resDF[c].astype('Float32')
 if resDF[c].max() <65000:
resDF[c]= resDF[c].astype('Float16' )<merge>
house_data_test[house_data_test.isnull().any(axis=1)]
grpByItem= generateGroupByFeatures(salesData,['item_id'],{'item_price':['min','max',np.var]})
 grpByItem['historic_price_shift']=(grpByItem['item_price_max']-grpByItem['item_price_min'])/grpByItem['item_price_max']
grpByItem.drop(['item_price_max','item_price_min'],axis=1,inplace=True)
 resDF= resDF.merge(grpByItem,how='left',on=['item_id'] )<merge>
train=house_data
test=house_data_test
train['train']=1
test['train']=0
combined=pd.concat([train,test] )
%%time
for m in meanEncodingCols:
grpCols= m[0]
lagArr= m[1]
tmpDF= resDF.copy()
grpCols= grpCols+['date_block_num']
groupDF= generateGroupByFeatures(tmpDF,grpCols,{'target':['mean']})
 tmpDF= tmpDF.merge(groupDF,how='left',on= grpCols)
 meanEncodedColName= [c for c in groupDF.columns if 'target_mean' in c]
assert len(meanEncodedColName)==1
meanEncodedColName= meanEncodedColName[0]
for lag in lagArr:
shiftedDF= tmpDF[['item_id','shop_id','date_block_num',meanEncodedColName]].copy()
shiftedDF['date_block_num']= shiftedDF['date_block_num']+lag
lagColName= meanEncodedColName+"_lag_"+str(lag)
 shiftedDF.rename({meanEncodedColName:lagColName},axis=1,inplace=True)
 resDF= resDF.merge(shiftedDF,how='left',on=['item_id','shop_id','date_block_num'])
 del shiftedDF
del tmpDF<merge>
ohe_data_frame=pd.get_dummies(combined,
columns=['MSZoning', 'Street', 'LotShape', 'LandContour', 'Utilities',
'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2',
'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st',
'Exterior2nd', 'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',
'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'KitchenQual',
'Functional', 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond',
'PavedDrive', 'SaleType', 'SaleCondition'],
)
%%time
for lag in [1,2,3]:
tmpDF= resDF[['item_id','date_block_num','avg_price']].copy()
tmpDF['date_block_num']+= lag
tmpGrpByItem= tmpDF.groupby(['item_id','date_block_num'],as_index=False ).agg({'avg_price':['mean']})
 tmpGrpByItem.columns= ['item_id','date_block_num','item_price_lag'+str(lag)]
resDF= resDF.merge(tmpGrpByItem,how='left',on=['item_id','date_block_num'])
 del tmpGrpByItem
del tmpDF<merge>
train_df=ohe_data_frame[ohe_data_frame['train']==1]
test_df=ohe_data_frame[ohe_data_frame['train']==0]
train_df.drop(['train'],axis=1,inplace=True)
 test_df.drop(['train','SalePrice'],axis=1,inplace=True )
%%time
for lag in [1,2,3]:
tmpDF= resDF[['item_category_id','date_block_num','avg_price']].copy()
tmpDF['date_block_num']+= lag
tmpGrpByItem= tmpDF.groupby(['item_category_id','date_block_num'],as_index=False ).agg({'avg_price':['mean']})
 tmpGrpByItem.columns= ['item_category_id','date_block_num','item_cat_price_lag'+str(lag)]
resDF= resDF.merge(tmpGrpByItem,how='left',on=['item_category_id','date_block_num'])
 del tmpGrpByItem
del tmpDF<count_missing_values>
house_data=train_df
house_data_test=test_df
for col in resDF.columns:
if 'price' not in col and(resDF[col].isnull().any()):
resDF[col].fillna(0, inplace=True )<data_type_conversions>
X_train = house_data.drop('SalePrice', axis=1)
 Y_train = house_data['SalePrice']
X_test = house_data_test
for col in resDF.columns:
if 'price' in col:
resDF[col]= resDF[col].fillna(resDF[col].median() )<load_pretrained>
gc.collect()
resDF.to_pickle('data.pkl' )<feature_engineering>
params = {'n_estimators': 3000, 'max_depth': 1, 'min_samples_leaf':15, 'min_samples_split':10,
'learning_rate': 0.05, 'loss': 'huber','max_features':'sqrt'}
gbr_model = GradientBoostingRegressor(**params)
 gbr_model.fit(X_train, Y_train )
resDF['month']= resDF['date_block_num']%12
resDF=resDF[ resDF['date_block_num']>12 ]<drop_column>
gbr_model.score(X_train, Y_train )
for c in ['item_price_lag1', 'item_price_lag2','item_price_lag3']:
resDF[c+'_diff']= resDF[c]/resDF['avg_price']
resDF.drop([c],axis=1,inplace=True )<prepare_x_and_y>
y_grad_predict = gbr_model.predict(X_test)
 print(y_grad_predict )
<prepare_x_and_y><EOS> 
my_submission = pd.DataFrame({'Id': test_parent.Id, 'SalePrice': y_grad_predict})
 print(my_submission)
 my_submission.to_csv('submission.csv', encoding='utf-8', index=False )