Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<concatenate>
warnings.filterwarnings('ignore')
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
test = pd.read_csv('.. /input/ashrae-energy-prediction/test.csv', index_col=0, parse_dates = ['timestamp'])
 building = pd.read_csv('.. /input/ashrae-energy-prediction/building_metadata.csv', usecols=['site_id', 'building_id'] )
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
<feature_engineering>
test = test.merge(building, left_on = "building_id", right_on = "building_id", how = "left")
 t = test[['building_id', 'meter', 'timestamp']]
t['row_id'] = t.index
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
submission_base = pd.read_csv('.. /input/ashrae-half-and-half/submission.csv', index_col=0 )
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
submission = submission_base.copy()
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
site_0 = pd.read_csv('.. /input/new-ucf-starter-kernel/submission_ucf_replaced.csv', index_col=0)
 submission.loc[test[test['site_id']==0].index, 'meter_reading'] = site_0['meter_reading']
del site_0
gc.collect()
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
<feature_engineering>
with open('.. /usr/lib/ucl_data_leakage_episode_2/site1.pkl', 'rb')as f:
site_1 = pickle.load(f)
 site_1 = site_1[site_1['timestamp'].dt.year > 2016]
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
site_1 = site_1.merge(t, left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 site_1 = site_1[['meter_reading_scraped', 'row_id']].set_index('row_id' ).dropna()
submission.loc[site_1.index, 'meter_reading'] = site_1['meter_reading_scraped']
del site_1
gc.collect()
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<compute_test_metric>
site_2 = pd.read_csv('.. /input/asu-buildings-energy-consumption/asu_2016-2018.csv', parse_dates = ['timestamp'])
 site_2 = site_2[site_2['timestamp'].dt.year > 2016]
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<prepare_x_and_y>
site_2 = site_2.merge(t, left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 site_2 = site_2[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
submission.loc[site_2.index, 'meter_reading'] = site_2['meter_reading']
del site_2
gc.collect()
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))<choose_model_class>
site_4 = pd.read_csv('.. /input/ucb-data-leakage-site-4/site4.csv', parse_dates = ['timestamp'])
 site_4.columns = ['building_id', 'timestamp', 'meter_reading']
site_4['meter'] = 0
site_4['timestamp'] = pd.DatetimeIndex(site_4['timestamp'])+ timedelta(hours=-8)
 site_4 = site_4[site_4['timestamp'].dt.year > 2016]
NFOLD = 5
kf = KFold(n_splits=NFOLD )<predict_on_test>
site_4 = site_4.merge(t, left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 site_4 = site_4[['meter_reading', 'row_id']].dropna().set_index('row_id')
 submission.loc[site_4.index, 'meter_reading'] = site_4['meter_reading']
del site_4
gc.collect()
<compute_test_metric><EOS> 
submission.to_csv('submission.csv' )