Kaggle competition: ASHRAE - Great Energy Predictor III
feature_names = ['Team', 'X', 'Y', 'S', 'A', 'Dis', 'Orientation', 'Dir', 'NflId',
'Season', 'YardLine', 'Quarter', 'PossessionTeam', 'Down', 'Distance',
'FieldPosition', 'HomeScoreBeforePlay', 'VisitorScoreBeforePlay',
'NflIdRusher', 'OffenseFormation', 'DefendersInTheBox', 'PlayDirection',
'PlayerHeight', 'PlayerWeight', 'Position', 'HomeTeamAbbr',
'VisitorTeamAbbr', 'Week', 'Stadium', 'Location', 'StadiumType', 'Turf',
'GameWeather', 'Temperature', 'Humidity', 'WindSpeed', 'WindDirection',
'DefensePersonnel_DL', 'DefensePersonnel_LB', 'DefensePersonnel_DB',
'OffensePersonnel_Flag', 'OffensePersonnel_QB', 'OffensePersonnel_RB',
'OffensePersonnel_TE', 'OffensePersonnel_WR', 'OffensePersonnel_OL',
'TimeSnapHandoff', 'GameYear', 'GameMonth', 'GameDay', 'PlayerAgeYears',
'GameClockSeconds']
dfy = dfa_sc[['Yards']]
dfX = dfa_sc[feature_names]
yards = dfy.Yards
y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx][99 + target] = 1<split>
def find_bad_sitezero(X):

 return X[(X.timestamp < 3378)&(X.site_id == 0)&(X.meter == 0)].index
X_train, X_test, y_train, y_test = train_test_split(dfX, y, test_size=0.33, random_state = 8)
 X_train.shape,y_train.shape, X_test.shape, y_test.shape<compute_test_metric>
def find_bad_building1099(X, y):

 return X[(X.building_id == 1099)&(X.meter == 2)&(y > 3e4)].index
def crps(y_true, y_pred):
y_true = np.clip(np.cumsum(y_true, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 return(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * y_true.shape[0] )<choose_model_class>
def find_bad_rows(X, y):
return find_bad_zeros(X, y ).union(find_bad_sitezero(X)).union(find_bad_building1099(X, y))
def build_model() :
model = keras.Sequential([
layers.Dense(64, activation='relu', input_shape=[X_train.shape[1]]),
layers.BatchNormalization() ,
layers.Dense(64, activation='relu'),
layers.BatchNormalization() ,
layers.Dense(199, activation='softmax')
 ])
 model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
 return model<choose_model_class>
pd.set_option("max_columns", 500)
 def input_file(file):
path = f".. /input/ashrae-energy-prediction/{file}"
if not os.path.exists(path): return path + ".gz"
return path
def compress_dataframe(df):
result = df.copy()
for col in result.columns:
col_data = result[col]
dn = col_data.dtype.name
if dn == "object":
result[col] = pd.to_numeric(col_data.astype("category" ).cat.codes, downcast="integer")
 elif dn == "bool":
result[col] = col_data.astype("int8")
 elif dn.startswith("int")or(col_data.round() == col_data ).all() :
result[col] = pd.to_numeric(col_data, downcast="integer")
 else:
result[col] = pd.to_numeric(col_data, downcast='float')
 return result
def read_train() :
df = pd.read_csv(input_file("train.csv"), parse_dates=["timestamp"])
 df.timestamp =(df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
return compress_dataframe(df)
 def read_building_metadata() :
return compress_dataframe(pd.read_csv(
 input_file("building_metadata.csv")).fillna(-1)).set_index("building_id")
 site_GMT_offsets = [-5, 0, -7, -5, -8, 0, -5, -5, -5, -6, -7, -5, 0, -6, -5, -5]
def read_weather_train(fix_timestamps=True, interpolate_na=True, add_na_indicators=True):
df = pd.read_csv(input_file("weather_train.csv"), parse_dates=["timestamp"])
 df.timestamp =(df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
if fix_timestamps:
GMT_offset_map = {site: offset for site, offset in enumerate(site_GMT_offsets)}
df.timestamp = df.timestamp + df.site_id.map(GMT_offset_map)
 if interpolate_na:
site_dfs = []
for site_id in df.site_id.unique() :
site_df = df[df.site_id == site_id].set_index("timestamp" ).reindex(range(8784))
site_df.site_id = site_id
for col in [c for c in site_df.columns if c != "site_id"]:
if add_na_indicators: site_df[f"had_{col}"] = ~site_df[col].isna()
site_df[col] = site_df[col].interpolate(limit_direction='both', method='linear')
 site_df[col] = site_df[col].fillna(df[col].median())
 site_dfs.append(site_df)
 df = pd.concat(site_dfs ).reset_index()
elif add_na_indicators:
for col in df.columns:
if df[col].isna().any() : df[f"had_{col}"] = ~df[col].isna()
return compress_dataframe(df ).set_index(["site_id", "timestamp"])
 def combined_train_data(fix_timestamps=True, interpolate_na=True, add_na_indicators=True):
Xy = compress_dataframe(read_train().join(read_building_metadata() , on="building_id" ).join(
 read_weather_train(fix_timestamps, interpolate_na, add_na_indicators),
on=["site_id", "timestamp"] ).fillna(-1))
return Xy.drop(columns=["meter_reading"]), Xy.meter_reading
def _add_time_features(X):
return X.assign(tm_day_of_week=(( X.timestamp // 24)% 7), tm_hour_of_day=(X.timestamp % 24))
class CatSplitRegressor(BaseEstimator, RegressorMixin):
def __init__(self, model, col):
self.model = model
self.col = col
def fit(self, X, y):
self.fitted = {}
importances = []
for val in X[self.col].unique() :
X1 = X[X[self.col] == val].drop(columns=[self.col])
 self.fitted[val] = clone(self.model ).fit(X1, y.reindex_like(X1))
importances.append(self.fitted[val].feature_importances_)
 del X1
fi = np.average(importances, axis=0)
 col_index = list(X.columns ).index(self.col)
 self.feature_importances_ = [*fi[:col_index], 0, *fi[col_index:]]
return self
def predict(self, X):
result = np.zeros(len(X))
for val in X[self.col].unique() :
ix = np.nonzero(( X[self.col] == val ).to_numpy())
 predictions = self.fitted[val].predict(X.iloc[ix].drop(columns=[self.col]))
result[ix] = predictions
return result
categorical_columns = [
"building_id", "meter", "site_id", "primary_use", "had_air_temperature", "had_cloud_coverage",
"had_dew_temperature", "had_precip_depth_1_hr", "had_sea_level_pressure", "had_wind_direction",
"had_wind_speed", "tm_day_of_week", "tm_hour_of_day"
]
class LGBMWrapper(BaseEstimator, RegressorMixin):
def __init__(self, categorical_feature=None, **params):
self.model = LGBMRegressor(**params)
 self.categorical_feature = categorical_feature
def fit(self, X, y):
with warnings.catch_warnings() :
cats = None if self.categorical_feature is None else list(
 X.columns.intersection(self.categorical_feature))
warnings.filterwarnings("ignore",
"categorical_feature in Dataset is overridden".lower())
 self.model.fit(X, y, **({} if cats is None else {"categorical_feature": cats}))
self.feature_importances_ = self.model.feature_importances_
return self
def predict(self, X):
return self.model.predict(X)
 def get_params(self, deep=True):
return {**self.model.get_params(deep), "categorical_feature": self.categorical_feature}
def set_params(self, **params):
ctf = params.pop("categorical_feature", None)
 if ctf is not None: self.categorical_feature = ctf
self.model.set_params(params )
model = build_model()<choose_model_class>
def read_test() :
df = pd.read_csv(input_file("test.csv"), parse_dates=["timestamp"])
 df.timestamp =(df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
return compress_dataframe(df ).set_index("row_id")
 def read_weather_test(fix_timestamps=True, interpolate_na=True, add_na_indicators=True):
df = pd.read_csv(input_file("weather_test.csv"), parse_dates=["timestamp"])
 df.timestamp =(df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
if fix_timestamps:
GMT_offset_map = {site: offset for site, offset in enumerate(site_GMT_offsets)}
df.timestamp = df.timestamp + df.site_id.map(GMT_offset_map)
 if interpolate_na:
site_dfs = []
for site_id in df.site_id.unique() :
site_df = df[df.site_id == site_id].set_index("timestamp" ).reindex(range(8784, 26304))
site_df.site_id = site_id
for col in [c for c in site_df.columns if c != "site_id"]:
if add_na_indicators: site_df[f"had_{col}"] = ~site_df[col].isna()
site_df[col] = site_df[col].interpolate(limit_direction='both', method='linear')
 site_df[col] = site_df[col].fillna(df[col].median())
 site_dfs.append(site_df)
 df = pd.concat(site_dfs ).reset_index()
elif add_na_indicators:
for col in df.columns:
if df[col].isna().any() : df[f"had_{col}"] = ~df[col].isna()
return compress_dataframe(df ).set_index(["site_id", "timestamp"])
 def combined_test_data(fix_timestamps=True, interpolate_na=True, add_na_indicators=True):
X = compress_dataframe(read_test().join(read_building_metadata() , on="building_id" ).join(
 read_weather_test(fix_timestamps, interpolate_na, add_na_indicators),
on=["site_id", "timestamp"] ).fillna(-1))
return X
early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10 )<train_model>
X, y = combined_train_data()
bad_rows = find_bad_rows(X, y)
 pd.Series(bad_rows.sort_values() ).to_csv("rows_to_drop.csv", header=False, index=False )
history = model.fit(X_train,
y_train,
epochs=40,
batch_size=512,
validation_data=(X_test, y_test),
verbose=1,
callbacks=[early_stop] )<prepare_output>
X = X.drop(index=bad_rows)
 y = y.reindex_like(X)
 X = compress_dataframe(_add_time_features(X))
X = X.drop(columns="timestamp")
 y = np.log1p(y)
 model = CatSplitRegressor(
 LGBMWrapper(random_state=0, n_jobs=-1, categorical_feature=categorical_columns), "meter")
 model.fit(X, y)
 del X, y
hist = pd.DataFrame(history.history)
 hist['epoch'] = history.epoch
hist.tail()<predict_on_test>
X = combined_test_data()
X = compress_dataframe(_add_time_features(X))
X = X.drop(columns="timestamp")
 predictions = pd.DataFrame({
"row_id": X.index,
"meter_reading": np.clip(np.expm1(model.predict(X)) , 0, None)
 })
 del X
<compute_test_metric><EOS> 
predictions.to_csv("submission.csv", index=False, float_format="%.4f" )