Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<drop_column>
DATA_PATH = ".. /input/ashrae-energy-prediction/"
train.drop(train.index[[523, 691, 1169, 1182, 1298]] )<concatenate>
train_df = pd.read_csv(DATA_PATH + 'train.csv')
 train_df = train_df [ train_df['building_id'] != 1099 ]
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")')
 building_df = pd.read_csv(DATA_PATH + 'building_metadata.csv')
 weather_df = pd.read_csv(DATA_PATH + 'weather_train.csv' )
all_data = pd.concat(( train, test)).reset_index(drop = True )<count_values>
def fill_weather_dataset(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
start_date = datetime.datetime.strptime(weather_df['timestamp'].min() ,time_format)
 end_date = datetime.datetime.strptime(weather_df['timestamp'].max() ,time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
missing_hours = []
for site_id in range(16):
site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
 new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])
 new_rows['site_id'] = site_id
weather_df = pd.concat([weather_df,new_rows])
 weather_df = weather_df.reset_index(drop=True)
 weather_df["datetime"] = pd.to_datetime(weather_df["timestamp"])
 weather_df["day"] = weather_df["datetime"].dt.day
weather_df["week"] = weather_df["datetime"].dt.week
weather_df["month"] = weather_df["datetime"].dt.month
weather_df = weather_df.set_index(['site_id','day','month'])
 air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean() ,columns=["air_temperature"])
 weather_df.update(air_temperature_filler,overwrite=False)
 cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()
cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=["cloud_coverage"])
 weather_df.update(cloud_coverage_filler,overwrite=False)
 due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean() ,columns=["dew_temperature"])
 weather_df.update(due_temperature_filler,overwrite=False)
 sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()
sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])
 weather_df.update(sea_level_filler,overwrite=False)
 wind_direction_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean() ,columns=['wind_direction'])
 weather_df.update(wind_direction_filler,overwrite=False)
 wind_speed_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean() ,columns=['wind_speed'])
 weather_df.update(wind_speed_filler,overwrite=False)
 precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()
precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])
 weather_df.update(precip_depth_filler,overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)
 return weather_df
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df
def features_engineering(df):
df.sort_values("timestamp")
 df.reset_index(drop=True)
 df["timestamp"] = pd.to_datetime(df["timestamp"],format="%Y-%m-%d %H:%M:%S")
 df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df['square_feet'] = np.log1p(df['square_feet'])
 drop = ["timestamp","sea_level_pressure", "wind_direction", "wind_speed","year_built","floor_count"]
df = df.drop(drop, axis=1)
 gc.collect()
le = LabelEncoder()
df["primary_use"] = le.fit_transform(df["primary_use"])
 return df
for feature in all_data.select_dtypes(exclude=['object'] ).columns:
all_data[feature].fillna(all_data[feature].median() , inplace = True)
 for feature in all_data.select_dtypes(include=['object'] ).columns:
all_data[feature].fillna(all_data[feature].value_counts().idxmax() , inplace = True )<feature_engineering>
weather_df = fill_weather_dataset(weather_df )
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
all_data[skewed_feats] = np.log1p(all_data[skewed_feats] )<define_variables>
train_df = reduce_mem_usage(train_df,use_float16=True)
 building_df = reduce_mem_usage(building_df,use_float16=True)
 weather_df = reduce_mem_usage(weather_df,use_float16=True )
f_drop_column = [
'BsmtFinSF1',
'LotFrontage',
'WoodDeckSF',
'2ndFlrSF',
'OpenPorchSF',
'HalfBath',
'LotArea',
'BsmtFullBath',
'BsmtUnfSF',
'BedroomAbvGr',
'ScreenPorch',
'PoolArea',
'MoSold',
'3SsnPorch',
'BsmtFinSF2',
'BsmtHalfBath',
'MiscVal',
'Id',
'LowQualFinSF',
'YrSold',
'OverallCond',
'MSSubClass',
'EnclosedPorch',
'KitchenAbvGr'
]<define_variables>
train_df = train_df.merge(building_df, left_on='building_id',right_on='building_id',how='left')
 train_df = train_df.merge(weather_df,how='left',left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])
 del weather_df
gc.collect()
MSZoning_map = {'RL':1, 'RM':0, 'C(all)':0, 'FV':1, 'RH':0}
Street_map = {"Pave":1, "Grvl":1}
Alley_map = {"Grvl":1, "Pave":1}
LotShape_map = {"Reg":0, "IR1":1, "IR2":1, "IR3":1}
LandContour_map = {"Lv1":0, "Bnk":0, "Low":1, "HLS":1}
Utilities_map = {"AllPub":1, "NoSeWa":1}
LotConfig_map = {"Inside":0, "FR2":0, "Corner":0, "CulDSac":1, "FR3":1}
LandSlope_map = {"Gtl":1, "Mod":1, "Sev":1}
Neighborhood_map = {
'CollgCr':1, 'Veenker':1, 'Crawfor':1, 'NoRidge':2, 'Mitchel':0, 'Somerst':1, 'NAmes':1, 'OldTown':0, 'BrkSide':0,
'Sawyer':0, 'NridgHt':2, 'NWAmes':0, 'SawyerW':1, 'IDOTRR':0, 'MeadowV':0, 'Edwards':0, 'Timber':1, 'Gilbert':1,
'StoneBr':2, 'ClearCr':1, 'NPkVill':0, 'Blmngtn':1, 'BrDale':0, 'SWISU':0, 'Blueste':0,
}
Condition1_map = {
"Norm":0, "Feedr":0, "PosN":1, "Artery":0, "RRAe":0, "RRNn":1, "RRAn":0, "PosA":1, "RRNe":0
}
Condition2_map = {
"Norm":0, "Artery":0, "RRNn":0, "Feedr":0, "PosN":1, "PosA":1, "RRAn":0, "RRAe":0,
}
BldgType_map = {
"1Fam":1, "2fmCon":1, "Duplex":1, "TwnhsE":1, "Twnhs":1
}
HouseStyle_map = {
'2Story':0, '1Story':0, '1.5Fin':0, '1.5Unf':0, 'SFoyer':0, 'SLvl':0, '2.5Unf':0, '2.5Fin':1
}
RoofStyle_map = {
'Gable':1, 'Hip':1, 'Gambrel':1, 'Mansard':1, 'Flat':1, 'Shed':1
}
RoofMatl_map = {
'CompShg':0, 'WdShngl':1, 'Metal':0, 'WdShake':0, 'Membran':0, 'Tar&Grv':0, 'Roll':0, 'ClyTile':0
}
Exterior1st_map = {
'VinylSd':1, 'MetalSd':0, 'Wd Sdng':0, 'HdBoard':0, 'BrkFace':1, 'WdShing':0,
'CemntBd':1, 'Plywood':0, 'AsbShng':0, 'Stucco':0, 'BrkComm':0, 'AsphShn':0,
'Stone':2, 'ImStucc':2, 'CBlock':0
}
Exterior2nd_map = {
'VinylSd':1, 'MetalSd':0, 'Wd Shng':0, 'HdBoard':0, 'Plywood':0, 'Wd Sdng':0,
'CmentBd':1, 'BrkFace':0, 'Stucco':0, 'AsbShng':0, 'Brk Cmn':0, 'ImStucc':1,
'AsphShn':0, 'Stone':0, 'Other':0, 'CBlock':0
}
MasVnrType_map = {
'BrkFace':0, 'None':0, 'Stone':1, 'BrkCmn':0
}
ExterQual_map = {
'Gd':0, 'TA':0, 'Ex':1, 'Fa':0
}
ExterCond_map = {
'TA':0, 'Gd':0, 'Fa':0, 'Po':0, 'Ex':1
}
Foundation_map = {
'PConc':1, 'CBlock':0, 'BrkTil':0, 'Wood':1, 'Slab':0, 'Stone':1
}
BsmtQual_map = {
'Gd':0, 'TA':0, 'Ex':1, 'Fa':0
}
BsmtCond_map = {
'TA':1, 'Gd':1, 'Fa':0, 'Po':0
}
BsmtExposure_map = {
'No':0, 'Gd':1, 'Mn':0, 'Av':0
}
BsmtFinType1_map = {
'GLQ':1, 'ALQ':0, 'Unf':0, 'Rec':0, 'BLQ':0, 'LwQ':0
}
BsmtFinType2_map = {
'Unf':0, 'BLQ':0, 'ALQ':1, 'Rec':0, 'LwQ':0, 'GLQ':1
}
Heating_map = {
'GasA':1, 'GasW':1, 'Grav':0, 'Wall':0, 'OthW':0, 'Floor':0
}
HeatingQC_map = {
'Ex':1, 'Gd':0, 'TA':0, 'Fa':0, 'Po':0
}
CentralAir_map = {
'Y':1, 'N':0
}
Electrical_map = {
'SBrkr':1, 'FuseF':0, 'FuseA':0, 'FuseP':0, 'Mix':0
}
KitchenQual_map = {
'Gd':0, 'TA':0, 'Ex':1, 'Fa':0
}
Functional_map = {
'Typ':1, 'Min1':1, 'Maj1':1, 'Min2':1, 'Mod':1, 'Maj2':0, 'Sev':1
}
FireplaceQu_map = {
'Gd':0, 'TA':0, 'Fa':0, 'Ex':1, 'Po':0
}
GarageType_map = {
'Attchd':1, 'Detchd':0, 'BuiltIn':1, 'CarPort':0, 'Basment':0, '2Types':0
}
GarageFinish_map = {
'RFn':0, 'Unf':0, 'Fin':1
}
GarageQual_map = {
'TA':0, 'Fa':0, 'Gd':1, 'Ex':1, 'Po':0
}
GarageCond_map = {
'TA':1, 'Fa':0, 'Gd':0, 'Po':0, 'Ex':0
}
PavedDrive_map = {
'Y':1, 'N':1, 'P':1
}
PoolQC_map = {
'Ex':1, 'Fa':0, 'Gd':0
}
Fence_map = {
'MnPrv':1, 'GdWo':1, 'GdPrv':1, 'MnWw':1
}
MiscFeature_map = {
'Shed':1, 'Gar2':1, 'Othr':0, 'TenC':1
}
SaleType_map = {
'WD':0, 'New':1, 'COD':0, 'ConLD':0, 'ConLI':0, 'CWD':0, 'ConLw':0, 'Con':1, 'Oth':0
}
SaleCondition_map = {
'Normal':0, 'Abnorml':0, 'Partial':1, 'AdjLand':0, 'Alloca':0, 'Family':0
}<categorify>
train_df = features_engineering(train_df )
all_data["MSZoning"] = all_data.MSZoning.replace(MSZoning_map)
 all_data["Street"] = all_data.Street.replace(Street_map)
 all_data["Alley"] = all_data.Alley.replace(Alley_map)
 all_data["LotShape"] = all_data.LotShape.replace(LotShape_map)
 all_data["LandContour"] = all_data.LandContour.replace(LandContour_map)
 all_data["Utilities"] = all_data.Utilities.replace(Utilities_map)
 all_data["LotConfig"] = all_data.LotConfig.replace(LotConfig_map)
 all_data["LandSlope"] = all_data.LandSlope.replace(LandSlope_map)
 all_data["Neighborhood"] = all_data.Neighborhood.replace(Neighborhood_map)
 all_data["Condition1"] = all_data.Condition1.replace(Condition1_map)
 all_data["Condition2"] = all_data.Condition2.replace(Condition2_map)
 all_data["BldgType"] = all_data.BldgType.replace(BldgType_map)
 all_data["HouseStyle"] = all_data.HouseStyle.replace(HouseStyle_map)
 all_data["RoofStyle"] = all_data.RoofStyle.replace(RoofStyle_map)
 all_data["RoofMatl"] = all_data.RoofMatl.replace(RoofMatl_map)
 all_data["Exterior1st"] = all_data.Exterior1st.replace(Exterior1st_map)
 all_data["Exterior2nd"] = all_data.Exterior2nd.replace(Exterior2nd_map)
 all_data["MasVnrType"] = all_data.MasVnrType.replace(MasVnrType_map)
 all_data["ExterQual"] = all_data.ExterQual.replace(ExterQual_map)
 all_data["ExterCond"] = all_data.ExterCond.replace(ExterCond_map)
 all_data["Foundation"] = all_data.Foundation.replace(Foundation_map)
 all_data["BsmtQual"] = all_data.BsmtQual.replace(BsmtQual_map)
 all_data["BsmtCond"] = all_data.BsmtCond.replace(BsmtCond_map)
 all_data["BsmtExposure"] = all_data.BsmtExposure.replace(BsmtExposure_map)
 all_data["BsmtFinType1"] = all_data.BsmtFinType1.replace(BsmtFinType1_map)
 all_data["BsmtFinType2"] = all_data.BsmtFinType2.replace(BsmtFinType2_map)
 all_data["Heating"] = all_data.Heating.replace(Heating_map)
 all_data["HeatingQC"] = all_data.HeatingQC.replace(HeatingQC_map)
 all_data["CentralAir"] = all_data.CentralAir.replace(CentralAir_map)
 all_data["Electrical"] = all_data.Electrical.replace(Electrical_map)
 all_data["KitchenQual"] = all_data.KitchenQual.replace(KitchenQual_map)
 all_data["Functional"] = all_data.Functional.replace(Functional_map)
 all_data["FireplaceQu"] = all_data.FireplaceQu.replace(FireplaceQu_map)
 all_data["GarageType"] = all_data.GarageType.replace(GarageType_map)
 all_data["GarageFinish"] = all_data.GarageFinish.replace(GarageFinish_map)
 all_data["GarageQual"] = all_data.GarageQual.replace(GarageQual_map)
 all_data["GarageCond"] = all_data.GarageCond.replace(GarageCond_map)
 all_data["PavedDrive"] = all_data.PavedDrive.replace(PavedDrive_map)
 all_data["PoolQC"] = all_data.PoolQC.replace(PoolQC_map)
 all_data["Fence"] = all_data.Fence.replace(Fence_map)
 all_data["MiscFeature"] = all_data.MiscFeature.replace(MiscFeature_map)
 all_data["SaleType"] = all_data.SaleType.replace(SaleType_map)
 all_data["SaleCondition"] = all_data.SaleCondition.replace(SaleCondition_map )<prepare_x_and_y>
target = np.log1p(train_df["meter_reading"])
 features = train_df.drop('meter_reading', axis = 1)
 del train_df
gc.collect()
train['SalePrice'] = np.log1p(train['SalePrice'])
 y = train['SalePrice']<categorify>
categorical_features = ["building_id", "site_id", "meter", "primary_use", "weekend"]
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.85,
"reg_lambda": 2,
"metric": "rmse",
}
kf = KFold(n_splits=3)
 models = []
for train_index,test_index in kf.split(features):
train_features = features.loc[train_index]
train_target = target.loc[train_index]
test_features = features.loc[test_index]
test_target = target.loc[test_index]
d_training = lgb.Dataset(train_features, label=train_target,categorical_feature=categorical_features, free_raw_data=False)
 d_test = lgb.Dataset(test_features, label=test_target,categorical_feature=categorical_features, free_raw_data=False)
 model = lgb.train(params, train_set=d_training, num_boost_round=1000, valid_sets=[d_training,d_test], verbose_eval=25, early_stopping_rounds=50)
 models.append(model)
 del train_features, train_target, test_features, test_target, d_training, d_test
gc.collect()
all_data = pd.get_dummies(all_data)
 all_data.shape<define_variables>
del features, target
gc.collect()
drop_columns = [
'LandSlope',
'Street',
'FireplaceQu',
'Alley',
'LandContour_0',
'Utilities',
'MiscFeature',
'PavedDrive',
'SaleCondition',
'Fence',
'HouseStyle',
'BldgType',
'Condition2',
'RoofStyle',
'LandContour_Lvl',
'Foundation',
'MasVnrArea',
'GarageYrBlt',
'TotRmsAbvGrd',
'Electrical',
'GarageArea',
'YearBuilt',
'GarageType',
'YearRemodAdd',
'LandContour_1',
] 
<drop_column>
test_df = pd.read_csv(DATA_PATH + 'test.csv')
 row_ids = test_df["row_id"]
test_df.drop("row_id", axis=1, inplace=True)
 test_df = reduce_mem_usage(test_df )
all_data.drop('SalePrice', axis=1, inplace=True )<prepare_x_and_y>
test_df = test_df.merge(building_df,left_on='building_id',right_on='building_id',how='left')
 del building_df
gc.collect()
train = all_data[:len(train)]
test = all_data[len(train):]
y = y<choose_model_class>
weather_df = pd.read_csv(DATA_PATH + 'weather_test.csv')
 weather_df = fill_weather_dataset(weather_df)
 weather_df = reduce_mem_usage(weather_df)
model_xgb = xgb.XGBRegressor(max_depth=2,learning_rate=0.05,n_estimators=1000, min_child_weight=3, reg_lambda=2 )<train_model>
test_df = test_df.merge(weather_df,how='left',on=['timestamp','site_id'])
 del weather_df
gc.collect()
model_xgb.fit(train, y )<find_best_model_class>
test_df = features_engineering(test_df )
def predict_cv(model, train_x, train_y, test_x):
preds = []
preds_test = []
va_indexes = []
kf = KFold(n_splits=4, shuffle=True, random_state=1234)
 for i,(tr_idx, va_idx)in enumerate(kf.split(train_x)) :
tr_x, va_x = train_x.iloc[tr_idx], train_x.iloc[va_idx]
tr_y, va_y = train_y.iloc[tr_idx], train_y.iloc[va_idx]
model.fit(tr_x, tr_y)
 tr_pred = model.predict(tr_x)
 pred = model.predict(va_x)
 preds.append(pred)
 pred_test = model.predict(test_x)
 preds_test.append(pred_test)
 va_indexes.append(va_idx)
 print(' score Train : {:.6f}'.format(np.sqrt(mean_squared_error(tr_y, tr_pred))),
' score Valid : {:.6f}'.format(np.sqrt(mean_squared_error(va_y, pred))))
va_indexes = np.concatenate(va_indexes)
 preds = np.concatenate(preds, axis=0)
 order = np.argsort(va_indexes)
 pred_train = pd.DataFrame(preds[order])
 preds_test = pd.DataFrame(np.mean(preds_test, axis=0))
print('Score : {:.6f}'.format(np.sqrt(mean_squared_error(train_y, pred_train))))
return pred_train, preds_test, model<predict_on_test>
results = []
for model in models:
if results == []:
results = np.expm1(model.predict(test_df, num_iteration=model.best_iteration)) / len(models)
 else:
results += np.expm1(model.predict(test_df, num_iteration=model.best_iteration)) / len(models)
 del model
gc.collect()
pred_train, preds_test, model = predict_cv(model_xgb, train, y, test)
<prepare_x_and_y>
del test_df, models
gc.collect()
<save_to_csv><EOS> 
results_df = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(results, 0, a_max=None)})
 del row_ids,results
gc.collect()
results_df.to_csv("submission.csv", index=False)