Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<drop_column>
warnings.filterwarnings('ignore')
 pd.set_option("max_columns", 500)
 %matplotlib inline
market_train1=market_train1.dropna()
rmv_len3 = np.abs(newer_len-market_train1.shape[0])
 print('There were %i lines removed additionally' %rmv_len3)
 deleted_rows=rmv_len1+rmv_len2+rmv_len3
print('There were %i lines in total removed' %deleted_rows )<feature_engineering>
train = pd.read_csv(".. /input/ashrae-energy-prediction/train.csv", parse_dates=["timestamp"])
 test = pd.read_csv(".. /input/ashrae-energy-prediction/test.csv", parse_dates=["timestamp"])
 building = pd.read_csv('.. /input/ashrae-energy-prediction/building_metadata.csv')
 weather_train = pd.read_csv('.. /input/ashrae-energy-prediction/weather_train.csv', parse_dates=["timestamp"])
 weather_test = pd.read_csv(".. /input/ashrae-energy-prediction/weather_test.csv", parse_dates=["timestamp"] )
market_train1['average'] =(market_train1['close'] + market_train1['open'])/2
market_train1['pricevolume'] = market_train1['volume'] * market_train1['close']<define_variables>
train_plot = train.merge(building, on='building_id', how='left')
 site = 0
meter_type = 1
primary_use = 'Education'
r = int(np.ceil(len(train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)]['building_id'].value_counts(dropna=False ).index.to_list())/2))
fig, axes = plt.subplots(r,2,figsize=(14, 36), dpi=100)
 for i, building_id in enumerate(train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)]['building_id'].value_counts(dropna=False ).index.to_list()):
train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)& 
(train_plot['building_id'] == building_id)][['timestamp', 'meter_reading']].set_index('timestamp' ).resample('H' ).mean() ['meter_reading'].plot(ax=axes[i%r][i//r],
alpha=0.8, label='By hour', color='tab:blue' ).set_ylabel('Mean meter reading', fontsize=13);
train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)& 
(train_plot['building_id'] == building_id)][['timestamp', 'meter_reading']].set_index('timestamp' ).resample('D' ).mean() ['meter_reading'].plot(ax=axes[i%r][i//r],
alpha=1, label='By day', color='tab:orange' ).set_xlabel('');
axes[i%r][i//r].legend() ;
axes[i%r][i//r].set_title('building_id: ' + str(building_id), fontsize=13);
plt.subplots_adjust(hspace=0.45)
 del train_plot,fig,axes,r
gc.collect() ;
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10','close_open_ratio','average','pricevolume']<create_dataframe>
def compress_dataframe(df):

 result = df.copy()
for col in result.columns:
col_data = result[col]
dn = col_data.dtype.name
if dn == "object":
result[col] = pd.to_numeric(col_data.astype("category" ).cat.codes, downcast="integer")
 elif dn == "bool":
result[col] = col_data.astype("int8")
 elif dn.startswith("int")or(col_data.round() == col_data ).all() :
result[col] = pd.to_numeric(col_data, downcast="integer")
 else:
result[col] = pd.to_numeric(col_data, downcast='float')
 return result
market_train=market_train1.copy()<split>
def set_time(df):
df.timestamp =(df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
return df
site_GMT_offsets = [-5, 0, -7, -5, -8, 0, -5, -5, -5, -6, -7, -5, 0, -6, -5, -5]
def weather_set_time(df,time_zone):
df.timestamp =(df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
GMT_offset_map = {site: offset for site, offset in enumerate(site_GMT_offsets)}
df.timestamp = df.timestamp + df.site_id.map(GMT_offset_map)
 site_dfs = []
for site_id in df.site_id.unique() :
site_df = df[df.site_id == site_id].set_index("timestamp" ).reindex(time_zone)
 site_df.site_id = site_id
for col in [c for c in site_df.columns if c != "site_id"]:
site_df[f"had_{col}"] = ~site_df[col].isna()
site_df[col] = site_df[col].interpolate(limit_direction='both', method='linear')
 site_df[col] = site_df[col].fillna(df[col].median())
 site_dfs.append(site_df)
 df = pd.concat(site_dfs ).reset_index()
for col in df.columns:
if df[col].isna().any() : df[f"had_{col}"] = ~df[col].isna()
return df
def _add_time_features(X):
return X.assign(tm_day_of_week=(( X.timestamp // 24)% 7), tm_hour_of_day=(X.timestamp % 24))
building = compress_dataframe(building.fillna(-1)).set_index("building_id")
 train = compress_dataframe(set_time(train))
test = compress_dataframe(set_time(test)).set_index("row_id")
 weather_train = compress_dataframe(weather_set_time(weather_train,range(8784)) ).set_index(["site_id", "timestamp"])
 weather_test = compress_dataframe(weather_set_time(weather_test,range(8784,26304)) ).set_index(["site_id", "timestamp"] )
train_indices, val_indices = train_test_split(market_train.index.values,test_size=0.25, random_state=23 )<normalization>
def combined_data(df,weather):
df = compress_dataframe(df.join(building, on="building_id" ).join(weather,
on=["site_id", "timestamp"] ).fillna(-1))
return df.drop(columns=["meter_reading"]),df.meter_reading
market_train[num_cols] = market_train[num_cols].fillna(0)
 print('scaling numerical columns')
 scaler = StandardScaler()
scaler = StandardScaler()
market_train[num_cols] = scaler.fit_transform(market_train[num_cols])
<choose_model_class>
def make_is_bad_zero(Xy_subset, min_interval=48, summer_start=3000, summer_end=7500):
meter = Xy_subset.meter_id.iloc[0]
is_zero = Xy_subset.meter_reading == 0
if meter == 0:
return is_zero
transitions =(is_zero != is_zero.shift(1))
all_sequence_ids = transitions.cumsum()
ids = all_sequence_ids[is_zero].rename("ids")
 if meter in [2, 3]:
keep = set(ids[(Xy_subset.timestamp < summer_start)| 
(Xy_subset.timestamp > summer_end)].unique())
 is_bad = ids.isin(keep)&(ids.map(ids.value_counts())>= min_interval)
 elif meter == 1:
time_ids = ids.to_frame().join(Xy_subset.timestamp ).set_index("timestamp" ).ids
is_bad = ids.map(ids.value_counts())>= min_interval
jan_id = time_ids.get(0, False)
 dec_id = time_ids.get(8283, False)
 if(jan_id and dec_id and jan_id == time_ids.get(500, False)and
dec_id == time_ids.get(8783, False)) :
is_bad = is_bad &(~(ids.isin(set([jan_id, dec_id]))))
else:
raise Exception(f"Unexpected meter type: {meter}")
 result = is_zero.copy()
result.update(is_bad)
 return result
def find_bad_zeros(X, y):

 Xy = X.assign(meter_reading=y, meter_id=X.meter)
 is_bad_zero = Xy.groupby(["building_id", "meter"] ).apply(make_is_bad_zero)
 return is_bad_zero[is_bad_zero].index.droplevel([0, 1])
 def find_bad_sitezero(X):

 return X[(X.timestamp < 3378)&(X.site_id == 0)&(X.meter == 0)].index
def find_bad_building1099(X, y):

 return X[(X.building_id == 1099)&(X.meter == 2)&(y > 3e4)].index
def find_bad_rows(X, y):
return find_bad_zeros(X, y ).union(find_bad_sitezero(X)).union(find_bad_building1099(X, y))
categorical_inputs = []
for cat in cat_cols:
categorical_inputs.append(Input(shape=[1], name=cat))
categorical_embeddings = []
for i, cat in enumerate(cat_cols):
categorical_embeddings.append(Embedding(embed_sizes[i], 10 )(categorical_inputs[i]))
categorical_logits = Flatten()(categorical_embeddings[0])
 categorical_logits = Dense(32,activation='relu' )(categorical_logits)
 a=len(num_cols)
 numerical_inputs = Input(shape=(a,), name='num')
 numerical_logits = numerical_inputs
numerical_logits = BatchNormalization()(numerical_logits)
 numerical_logits = Dense(128,activation='relu' )(numerical_logits)
 numerical_logits = Dense(64,activation='relu' )(numerical_logits)
 logits = Concatenate()([numerical_logits,categorical_logits])
 logits = Dense(64,activation='relu' )(logits)
 out = Dense(1, activation='sigmoid' )(logits)
 model = Model(inputs = categorical_inputs + [numerical_inputs], outputs=out)
 model.compile(optimizer='adam',loss=binary_crossentropy )<prepare_x_and_y>
X, y = combined_data(train,weather_train)
 bad_rows = find_bad_rows(X, y)
 pd.Series(bad_rows.sort_values() ).to_csv("rows_to_drop.csv", header=False, index=False)
 X = X.drop(index=bad_rows)
 y = y.reindex_like(X)
 X = _add_time_features(X)
 X = compress_dataframe(X)
 X = X.drop(columns="timestamp")
 del bad_rows,train,weather_train
gc.collect() ;
def get_input(market_train, indices):
X_num = market_train.loc[indices, num_cols].values
X = {'num':X_num}
for cat in cat_cols:
X[cat] = market_train.loc[indices, cat_cols].values
y =(market_train.loc[indices,'returnsOpenNextMktres10']>=0 ).values
r = market_train.loc[indices,'returnsOpenNextMktres10'].values
u = market_train.loc[indices, 'universe']
d = market_train.loc[indices, 'time'].dt.date
return X,y,r,u,d
X_train,y_train,r_train,u_train,d_train = get_input(market_train, train_indices)
 X_valid,y_valid,r_valid,u_valid,d_valid = get_input(market_train, val_indices )<choose_model_class>
y = np.log1p(y )
class SnapshotCallbackBuilder:
def __init__(self, nb_epochs, nb_snapshots, init_lr=0.1):
self.T = nb_epochs
self.M = nb_snapshots
self.alpha_zero = init_lr
def get_callbacks(self, model_prefix='Model'):
callback_list = [
callbacks.ModelCheckpoint("model.hdf5",monitor='val_my_iou_metric',
mode = 'max', save_best_only=True, verbose=1),
swa,
callbacks.LearningRateScheduler(schedule=self._cosine_anneal_schedule)
 ]
return callback_list
def _cosine_anneal_schedule(self, t):
cos_inner = np.pi *(t %(self.T // self.M))
cos_inner /= self.T // self.M
cos_out = np.cos(cos_inner)+ 1
return float(self.alpha_zero / 2 * cos_out )<train_model>
params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'eval_metric': 'rmse',
'num_leaves': 40,
'subsample':0.8,
'learning_rate': 0.03,
'verbose': 1,
'reg_lambda':3
}
num_trees = 1000
categorical_features=['building_id', 'site_id', 'primary_use', 'had_air_temperature', 'had_cloud_coverage',
'had_dew_temperature', 'had_precip_depth_1_hr','had_sea_level_pressure', 'had_wind_direction',
'had_wind_speed', 'tm_day_of_week', 'tm_hour_of_day']
check_point = ModelCheckpoint('model.hdf5',verbose=True, save_best_only=True)
 early_stop = EarlyStopping(patience=5,verbose=True)
 model.fit(X_train,y_train.astype(int),
validation_data=(X_valid,y_valid.astype(int)) ,
epochs=5,
verbose=True,
callbacks=[early_stop,check_point] )<compute_test_metric>
n_splits = 3
for val in X['meter'].unique() :
X1 = X[X['meter'] == val].drop(columns=['meter'])
 kf = StratifiedKFold(n_splits=n_splits,random_state=42)
 t = 0
for train_index, test_index in kf.split(X1, X1['tm_hour_of_day']):
train_features = X1.iloc[train_index]
train_target = y[X1.iloc[train_index].index]
test_features = X1.iloc[test_index]
test_target = y[X1.iloc[test_index].index]
d_train = lgb.Dataset(train_features, train_target, categorical_feature=categorical_features)
 d_eval = lgb.Dataset(test_features,test_target, categorical_feature=categorical_features)
 print("Building model meter :",val,'fold:',t)
 md=lgb.LGBMRegressor(boosting_type='gbdt', objective = 'regression',eval_metric='rmse',num_leaves=40,
n_estimators=num_trees, subsample=0.8, learning_rate=0.03,verbose=1,reg_lambda=3)
 md.fit(train_features,train_target,eval_set=(test_features, test_target),
early_stopping_rounds=200,verbose=20,categorical_feature=categorical_features,)
 locals() ['lgb_val{}_fold{}'.format(val,t)]=pickle.dumps(md)
 t += 1
del X1
del d_train, d_eval, train_features, test_features, md
gc.collect() ;
r_valid = r_valid.clip(-1,1)
 x_t_i = confidence_valid * r_valid * u_valid
data = {'day' : d_valid, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )<define_variables>
X = compress_dataframe(test.join(building, on="building_id" ).join(weather_test, on=["site_id", "timestamp"] ).fillna(-1))
X = compress_dataframe(_add_time_features(X))
X = X.drop(columns="timestamp")
 del test, weather_test
gc.collect() ;
<feature_engineering><EOS> 
result = np.zeros(len(X))
for val in X['meter'].unique() :
ix = np.nonzero(( X['meter'] == val ).to_numpy())
 for i in tqdm(range(n_splits)) :
model =pickle.loads(locals() ['lgb_val{}_fold{}'.format(val, i)])
 result[ix] += model.predict(X.iloc[ix].drop(columns=['meter']), num_iteration=model.best_iteration_)/n_splits
del model
gc.collect() ;
predictions = pd.DataFrame({
"row_id": X.index,
"meter_reading": np.clip(np.expm1(result), 0, None)
 })
 predictions.to_csv("submission.csv", index=False, float_format="%.4f" )