Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<import_modules>
warnings.filterwarnings('ignore')
train=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 train<sort_values>
test = pd.read_csv('.. /input/ashrae-energy-prediction/test.csv', index_col=0, parse_dates = ['timestamp'])
 building = pd.read_csv('.. /input/ashrae-energy-prediction/building_metadata.csv', usecols=['site_id', 'building_id'] )
k=train.isnull().sum()
k[k>0].sort_values(ascending=False)
<sort_values>
test = test.merge(building, left_on = "building_id", right_on = "building_id", how = "left" )
condition=condition.rename(columns={"SalePrice": "avg_sale_price_cond","Utilities": "total_records"})
 condition.sort_values(by='total_records',ascending=False,inplace=True)
 condition<feature_engineering>
submission_base = pd.read_csv('.. /input/ashrae-half-and-half/submission.csv', index_col=0 )
<feature_engineering>
submission = submission_base.copy()
<merge>
site_0 = pd.read_csv('.. /input/new-ucf-starter-kernel/submission_ucf_replaced.csv', index_col=0)
 gc.collect()
<feature_engineering>
site_0.to_csv('site0.csv.gz',index=False,compression='gzip', float_format='%.4f' )
neighborhood_md.loc[neighborhood_md['SalePrice']>=5000,'neighborhood_flag']=0
neighborhood_md.loc[neighborhood_md['SalePrice']>=150000,'neighborhood_flag']=1
neighborhood_md.loc[neighborhood_md['SalePrice']>=200000,'neighborhood_flag']=2
neighborhood_md.loc[neighborhood_md['SalePrice']>=250000,'neighborhood_flag']=3
del neighborhood_md['SalePrice']<merge>
FileLink('.. /input/new-ucf-starter-kernel/submission_ucf_replaced.csv' )
neighborhood_md.columns=['Neighborhood','neighborhood_flag']
train=pd.merge(train,neighborhood_md,left_on='Neighborhood',right_on='Neighborhood')
 test=pd.merge(test,neighborhood_md,left_on='Neighborhood',right_on='Neighborhood')
<feature_engineering>
with open('.. /usr/lib/ucl_data_leakage_episode_2/site1.pkl', 'rb')as f:
site_1 = pickle.load(f)
 site_1 = site_1[site_1['timestamp'].dt.year > 2016]
train['tot_bath']=train['BsmtFullBath']+0.5*train['BsmtHalfBath']+train['FullBath']+0.5*train['HalfBath']
train['bsmt_bath']=train['BsmtFullBath']+0.5*train['BsmtHalfBath']
train['bed_bath_kitch']=train['tot_bath']+train['BedroomAbvGr']+train['KitchenAbvGr']
train['area_floors']=train['1stFlrSF']+train['2ndFlrSF']+train['BsmtFinSF1']+train['BsmtFinSF2']
train['bsmt_by_total']=(train['BsmtFinSF1']+train['BsmtFinSF2'])/(train['area_floors'])
 train['unf_bsmt']=(train['BsmtUnfSF']/train['TotalBsmtSF'] ).fillna(0)
 train["unf_bsmt"].replace([np.inf, -np.inf], 0)
 train['porch_area_tot']=train['OpenPorchSF']+train['EnclosedPorch']+train['3SsnPorch']+train['ScreenPorch']
train['wood_deck_porch']=(train['WoodDeckSF']/train['porch_area_tot'] ).replace(np.inf, 0)
 train['sale_built_yr']=train['YrSold']-train['YearBuilt']
train['remod_built_yr']=train['YearRemodAdd']-train['YearBuilt']
train['new_flag']=0
train.loc[train['YrSold']-train['YearBuilt'],'new_flag']=1
train['remod_flag']=0
train.loc[train['remod_built_yr']>=2,'remod_flag']=1
train['floor_by_lot']=train['area_floors']/train['LotArea']
train['garage_area_per_car']=(train['GarageArea']/train['GarageCars'] ).fillna(0)
 train["garage_area_per_car"]=train["garage_area_per_car"].replace([np.inf, -np.inf], 0)
 test['tot_bath']=test['BsmtFullBath']+0.5*test['BsmtHalfBath']+test['FullBath']+0.5*test['HalfBath']
test['bsmt_bath']=test['BsmtFullBath']+0.5*test['BsmtHalfBath']
test['bed_bath_kitch']=test['tot_bath']+test['BedroomAbvGr']+test['KitchenAbvGr']
test['area_floors']=test['1stFlrSF']+test['2ndFlrSF']+test['BsmtFinSF1']+test['BsmtFinSF2']
test['bsmt_by_total']=(test['BsmtFinSF1']+test['BsmtFinSF2'])/(test['area_floors'])
 test['unf_bsmt']=(test['BsmtUnfSF']/test['TotalBsmtSF'] ).fillna(0)
 test["unf_bsmt"].replace([np.inf, -np.inf], 0)
 test['porch_area_tot']=test['OpenPorchSF']+test['EnclosedPorch']+test['3SsnPorch']+test['ScreenPorch']
test['wood_deck_porch']=(test['WoodDeckSF']/test['porch_area_tot'] ).replace(np.inf, 0)
 test['sale_built_yr']=test['YrSold']-test['YearBuilt']
test['remod_built_yr']=test['YearRemodAdd']-test['YearBuilt']
test['new_flag']=0
test.loc[test['YrSold']-test['YearBuilt']==0,'new_flag']=1
test['remod_flag']=0
test.loc[test['remod_built_yr']>=2,'remod_flag']=1
test['floor_by_lot']=test['area_floors']/test['LotArea']
test['garage_area_per_car']=(test['GarageArea']/test['GarageCars'] ).fillna(0)
 test["garage_area_per_car"]=test["garage_area_per_car"].replace([np.inf, -np.inf], 0 )<categorify>
site_1.to_csv('site1.csv.gz',index=False,compression='gzip', float_format='%.4f' )
mapping = { "NA" : 0,"Po" : 1,"Fa":2,"TA" : 3,"Gd":4,"Ex":5 }
mapping_shape = { "Reg" : 0,"IR1" : 1,"IR2":2,"IR3" : 3}
mapping_contour = { "Bnk" : 0,"Lvl" : 1,"Low":2,"HLS" : 3}
mapping_ms_zoning={"FV":4,"RL":3,"RH":2,"RM":1,"C(all)":0,"NA":-1}
mapping_paved_Drive={"Y":2,"P":1,"N":0}
mapping_utilities={"AllPub":1,"NoSeWa":0,"NoSeWr":0,"ELO":0,"NA":0}
mapping_functional={"Typ":4,"Min1":3,"Min2":3,"Mod":2,"Maj1":1,"Maj2":1,"Sev":0,"Sal":0,"NA":0}
l_col=['BsmtCond','BsmtQual','ExterQual','ExterCond','HeatingQC','GarageQual','GarageCond','FireplaceQu','KitchenQual']
l_col_zon=['MSZoning']
l_col_pav=['PavedDrive']
l_col_shape=['LotShape']
l_col_util=['Utilities']
l_col_con=['LandContour']
l_col_fun=['Functional']
def mapping_var(mapping,df,varlist):
for i in range(len(varlist)) :
varname=varlist[i]
varname_o=varname+'_o'
df[varname]=df[varname].fillna('NA')
 df[varname_o] =df[varname].apply(lambda x : mapping[x])
 return df
train=mapping_var(mapping,train,l_col)
 train=mapping_var(mapping_ms_zoning,train,l_col_zon)
 train=mapping_var(mapping_paved_Drive,train,l_col_pav)
 train=mapping_var(mapping_shape,train,l_col_shape)
 train=mapping_var(mapping_utilities,train,l_col_util)
 train=mapping_var(mapping_contour,train,l_col_con)
 train=mapping_var(mapping_functional,train,l_col_fun)
 test=mapping_var(mapping,test,l_col)
 test=mapping_var(mapping_ms_zoning,test,l_col_zon)
 test=mapping_var(mapping_paved_Drive,test,l_col_pav)
 test=mapping_var(mapping_shape,test,l_col_shape)
 test=mapping_var(mapping_utilities,test,l_col_util)
 test=mapping_var(mapping_contour,test,l_col_con)
 test=mapping_var(mapping_functional,test,l_col_fun)
 labelencoder = LabelEncoder()
train.loc[:, 'Street'] = labelencoder.fit_transform(train.loc[:, 'Street'])
 train.loc[:, 'CentralAir'] = labelencoder.fit_transform(train.loc[:, 'CentralAir'])
 test.loc[:, 'Street'] = labelencoder.fit_transform(test.loc[:, 'Street'])
 test.loc[:, 'CentralAir'] = labelencoder.fit_transform(test.loc[:, 'CentralAir'] )<feature_engineering>
t = test[['building_id', 'meter', 'timestamp']]
t['row_id'] = t.index
site_1 = site_1.merge(t, left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 site_1 = site_1[['meter_reading_scraped', 'row_id']].set_index('row_id' ).dropna()
submission.loc[site_1.index, 'meter_reading'] = site_1['meter_reading_scraped']
del site_1
gc.collect()
train['tot_quality']=train['BsmtQual_o']+train['ExterQual_o']+train['KitchenQual_o']+train['GarageQual_o']+train['FireplaceQu_o']
test['tot_quality']=test['BsmtQual_o']+test['ExterQual_o']+test['KitchenQual_o']+test['GarageQual_o']+test['FireplaceQu_o']<count_duplicates>
site_2 = pd.read_csv('.. /input/asu-buildings-energy-consumption/asu_2016-2018.csv', parse_dates = ['timestamp'])
 site_2 = site_2[site_2['timestamp'].dt.year > 2016]
k=corr.unstack().sort_values().drop_duplicates()
k[(k>0.8)|(k<-0.8)]<categorify>
site_2.to_csv('site2.csv.gz',index=False,compression='gzip', float_format='%.4f' )
<sort_values>
t = test[['building_id', 'meter', 'timestamp']]
t['row_id'] = t.index
site_2 = site_2.merge(t, left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 site_2 = site_2[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
submission.loc[site_2.index, 'meter_reading'] = site_2['meter_reading']
del site_2
gc.collect()
<categorify><EOS> 
submission.to_csv('submission.csv' )