Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<filter>
DATA_PATH = ".. /input/ashrae-energy-prediction/"
train[train["PlayId"] == 20170907000118]<count_unique_values>
train_df = pd.read_csv(DATA_PATH + 'train.csv')
 bad_rows = pd.read_csv('/kaggle/input/rows-to-drop/rows_to_drop.csv')
 train_df.drop(bad_rows.loc[:, '0'], inplace = True)
 train_df.reset_index(drop = True, inplace = True)
 building_df = pd.read_csv(DATA_PATH + 'building_metadata.csv')
 weather_df = pd.read_csv(DATA_PATH + 'weather_train.csv' )
train = cat_feat_processor.fit_transform(train)
 train = num_feat_processor.fit_transform(train)
 cat_features = []
int_features = []
float_features = []
bool_features = []
uint8_features = []
for col in train.columns:
if train[col].dtype =='object':
cat_features.append(( col, len(train[col].unique())))
 if train[col].dtype =='int64':
int_features.append(( col, len(train[col].unique())))
 if train[col].dtype =='float64':
float_features.append(( col, len(train[col].unique())))
 if train[col].dtype =='bool':
bool_features.append(( col, len(train[col].unique())))
 if train[col].dtype =='uint8':
uint8_features.append(( col, len(train[col].unique())))

print("{} columns in total".format(train.shape[1]))
print("{} Categorical Features found.".format(len(cat_features)))
 print("{} Integer Features found.".format(len(int_features)))
 print("{} Float Features found.".format(len(float_features)))
 print("{} Boolean Features found.".format(len(bool_features)))
 print("{} uint8 OneHot Features found.".format(len(uint8_features)))
 print("Integer Features:
{}".format(int_features))
print("Floating Features:
{}".format(float_features))
print("Boolean Features:
{}".format(bool_features))
print("Uint OneHot Features:
{}".format(uint8_features))<filter>
def fill_weather_dataset(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
start_date = datetime.datetime.strptime(weather_df['timestamp'].min() ,time_format)
 end_date = datetime.datetime.strptime(weather_df['timestamp'].max() ,time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
missing_hours = []
for site_id in range(16):
site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
 new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])
 new_rows['site_id'] = site_id
weather_df = pd.concat([weather_df,new_rows])
 weather_df = weather_df.reset_index(drop=True)
 weather_df["datetime"] = pd.to_datetime(weather_df["timestamp"])
 weather_df["day"] = weather_df["datetime"].dt.day
weather_df["week"] = weather_df["datetime"].dt.week
weather_df["month"] = weather_df["datetime"].dt.month
weather_df = weather_df.set_index(['site_id','day','month'])
 air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean() ,columns=["air_temperature"])
 weather_df.update(air_temperature_filler,overwrite=False)
 cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()
cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=["cloud_coverage"])
 weather_df.update(cloud_coverage_filler,overwrite=False)
 due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean() ,columns=["dew_temperature"])
 weather_df.update(due_temperature_filler,overwrite=False)
 sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()
sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])
 weather_df.update(sea_level_filler,overwrite=False)
 wind_direction_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean() ,columns=['wind_direction'])
 weather_df.update(wind_direction_filler,overwrite=False)
 wind_speed_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean() ,columns=['wind_speed'])
 weather_df.update(wind_speed_filler,overwrite=False)
 precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()
precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])
 weather_df.update(precip_depth_filler,overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)
 return weather_df
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df
def features_engineering(df):
df.sort_values("timestamp")
 df.reset_index(drop=True)
 df["timestamp"] = pd.to_datetime(df["timestamp"],format="%Y-%m-%d %H:%M:%S")
 df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df['square_feet'] = np.log1p(df['square_feet'])
 df['month'] = df['timestamp'].dt.month
df['month'].replace(( 1, 2, 3, 4), 1, inplace = True)
 df['month'].replace(( 5, 6, 7, 8), 2, inplace = True)
 df['month'].replace(( 9, 10, 11, 12), 3, inplace = True)
 drop = ["timestamp","sea_level_pressure", "wind_direction", "wind_speed","year_built","floor_count"]
df = df.drop(drop, axis=1)
 gc.collect()
le = LabelEncoder()
df["primary_use"] = le.fit_transform(df["primary_use"])
 return df
train[train["PlayId"] == 20170907000118]<count_values>
weather_df = fill_weather_dataset(weather_df )
train["CrossesWithin_1"].value_counts()<train_model>
train_df = reduce_mem_usage(train_df,use_float16=True)
 building_df = reduce_mem_usage(building_df,use_float16=True)
 weather_df = reduce_mem_usage(weather_df,use_float16=True )
class TrainingDataPreparation(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X):
return self
def transform(self, X):
X = X.sort_values(by=['PlayId', 'RusherTeam', 'IsRusher', 'dist_to_rusher'] ).reset_index()
unused_columns = ["GameId","PlayId","Team","IsRusher", "index"]
unique_columns = ["X", "Y", "S", "A", "X1sec", "Y1sec", "X3sec", "Y3sec", "RusherTeam", "CrossesWithin_1", "CrossesWithin_3", "dist_to_rusher", "Dis", "Orientation", "Dir", "JerseyNumber", "PlayerHeight", "PlayerWeight", "PlayerAge", "PlayerBMI"]
training_cols = []
for c in X.columns:
if c not in unique_columns + unused_columns:
training_cols.append(c)
 for c in unique_columns:
for i in range(22):
training_cols.append(c+str(i))
{}".format(len(training_cols), training_cols))
training_data=np.zeros(( X.shape[0]//22,len(training_cols)))
 for i in range(0,X.shape[0],22):
count=0
for c in training_cols:
if c in X:
training_data[i//22][count] = X[c][i]
count+=1
for c in unique_columns:
for j in range(22):
training_data[i//22][count] = X[c][i+j]
count+=1
X = pd.DataFrame(data=training_data, columns=training_cols)
 return X
train_dat_prep = TrainingDataPreparation()<train_model>
train_df = train_df.merge(building_df, left_on='building_id',right_on='building_id',how='left')
 train_df = train_df.merge(weather_df,how='left',left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])
 del weather_df
gc.collect()
y_train = train_transformed["Yards"]
X_train = train_transformed.drop(["Yards"], axis=1)
 print(y_train.shape)
 print(X_train.shape )<normalization>
train_df = features_engineering(train_df )
scaler_X = PowerTransformer()
X_train = scaler_X.fit_transform(X_train )<normalization>
target = np.log1p(train_df["meter_reading"])
 train_df = train_df.drop('meter_reading', axis = 1)
 gc.collect()
<compute_test_metric>
categorical_features = ["building_id", "site_id", "meter", "primary_use", "weekend"]
features = [col for col in train_df.columns if col not in ['month']]
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 1580,
"learning_rate": 0.05,
"feature_fraction": 0.85,
"reg_lambda": 1,
"metric": "rmse",
}
kf = StratifiedKFold(n_splits=5, shuffle=False, random_state=2319)
 models = []
for train_index,test_index in kf.split(train_df, train_df['month']):
train_features = train_df[features].loc[train_index]
train_target = target.loc[train_index]
test_features = train_df[features].loc[test_index]
test_target = target.loc[test_index]
d_training = lgb.Dataset(train_features, label=train_target,categorical_feature=categorical_features, free_raw_data=False)
 d_test = lgb.Dataset(test_features, label=test_target,categorical_feature=categorical_features, free_raw_data=False)
 model = lgb.train(params, train_set=d_training, num_boost_round=3000, valid_sets=[d_training,d_test], verbose_eval=25, early_stopping_rounds=50)
 models.append(model)
 del train_features, train_target, test_features, test_target, d_training, d_test
gc.collect()
def crps(y_true, y_pred):
return K.mean(K.square(y_true - K.cumsum(y_pred, axis=1)) , axis=1 )<choose_model_class>
del features, target
gc.collect()
def get_model() :
x = keras.layers.Input(shape=[X_train.shape[1]])
 fc1 = keras.layers.Dense(units=384, activation="relu", input_shape=[X_train.shape[1]] )(x)
 bn1 = keras.layers.BatchNormalization()(fc1)
 dp1 = keras.layers.Dropout(0.5 )(bn1)
 fc2 = keras.layers.Dense(units=256, activation="relu" )(dp1)
 bn2 = keras.layers.BatchNormalization()(fc2)
 dp2 = keras.layers.Dropout(0.5 )(bn2)
 output = keras.layers.Dense(units=199, activation='softmax' )(dp2)
 model = keras.models.Model(inputs=[x], outputs=[output])
 return model


earlyStopping = EarlyStopping(
 monitor='val_loss',
patience=15,
verbose=1,
mode='min',
restore_best_weights=True,
min_delta=1e-5 
)
 reduceLR = ReduceLROnPlateau(
 monitor='val_loss',
factor=0.5,
patience=8,
min_lr=1e-6,
verbose=1,
mode='min' 
)
 def train_model(X_train, y_train, X_val, y_val):
model = get_model()
model.compile(optimizer=Adam(lr=1e-4), loss=crps)
 checkPoint = ModelCheckpoint('best_model.h5',monitor='val_loss',mode='min', save_best_only=True, verbose=1, save_weights_only=True)
 model.fit(X_train, y_train, epochs=150, callbacks=[earlyStopping, reduceLR, checkPoint], validation_data=[X_val, y_val], batch_size=batch_size)
 return model<train_model>
test_df = pd.read_csv(DATA_PATH + 'test.csv')
 row_ids = test_df["row_id"]
test_df.drop("row_id", axis=1, inplace=True)
 test_df = reduce_mem_usage(test_df )
rkf = RepeatedKFold(n_splits=5, n_repeats=1)
 models = []
for tr_idx, vl_idx in rkf.split(X_train_nn, y_train_nn):
x_tr, y_tr = X_train_nn[tr_idx], y_train_nn[tr_idx]
x_vl, y_vl = X_train_nn[vl_idx], y_train_nn[vl_idx]
model = train_model(x_tr, y_tr, x_vl, y_vl)
 model.load_weights("best_model.h5")
 models.append(model )<prepare_x_and_y>
test_df = test_df.merge(building_df,left_on='building_id',right_on='building_id',how='left')
 del building_df
gc.collect()
def make_pred_nn(df, models):
tdata_local = cat_feat_processor.fit_transform(df)
 tdata_local = num_feat_processor.fit_transform(tdata_local)
 tdata_local = train_dat_prep.fit_transform(tdata_local)
 if("Yards" in tdata_local.columns):
tdata_local = tdata_local.drop(["Yards"], axis=1)
 tdata_local = scaler_X.transform(tdata_local)
 y_pred = np.zeros(( tdata_local.shape[0], 199))
for model in models:
y_pred += np.cumsum(model.predict(tdata_local), axis=1)
 y_pred = y_pred/len(models)
 return y_pred
def make_pred_env_nn(df, sample_prediction, env, models):
y_pred = make_pred_nn(df, models)
 env.predict(pd.DataFrame(data=y_pred.clip(0,1),columns=sample_prediction.columns))<load_from_csv>
weather_df = pd.read_csv(DATA_PATH + 'weather_test.csv')
 weather_df = fill_weather_dataset(weather_df)
 weather_df = reduce_mem_usage(weather_df )
<categorify>
test_df = test_df.merge(weather_df,how='left',on=['timestamp','site_id'])
 del weather_df
gc.collect()
<compute_test_metric>
test_df = features_engineering(test_df )
<predict_on_test>
results = []
for model in models:
if results == []:
results = np.expm1(model.predict(test_df, num_iteration=model.best_iteration)) / len(models)
 else:
results += np.expm1(model.predict(test_df, num_iteration=model.best_iteration)) / len(models)
 del model
gc.collect()
for test, sample_prediction in tqdm.tqdm(env.iter_test()):
make_pred_env_nn(test, sample_prediction, env, models )<save_to_csv>
del test_df, models
gc.collect()
env.write_submission_file()<import_modules>
results_df = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(results, 0, a_max=None)})
 del row_ids,results
gc.collect()
results_df.to_csv("submission.csv", index=False)