Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<choose_model_class>
import gc
import datetime
import pandas as pd
import numpy as np
import lightgbm as lgb
import os
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
learning_rate = kwargs.pop('lr', learning_rate)
 super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.learning_rate = K.variable(learning_rate, name='learning_rate')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(min_lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = K.maximum(self.total_steps - warmup_steps, 1)
 decay_rate =(self.min_lr - lr)/ decay_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr + decay_rate * K.minimum(t - warmup_steps, decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t))
self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t))
r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t >= 5, r_t * m_corr_t /(v_corr_t + self.epsilon), m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
@property
def lr(self):
return self.learning_rate
@lr.setter
def lr(self, learning_rate):
self.learning_rate = learning_rate
def get_config(self):
config = {
'learning_rate': float(K.get_value(self.learning_rate)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<compute_test_metric>
INPUT = ".. /input/ashrae-energy-prediction/"
df_train = pd.read_csv(f"{INPUT}train.csv")
 bldg_metadata = pd.read_csv(f"{INPUT}building_metadata.csv")
 weather_train = pd.read_csv(f"{INPUT}weather_train.csv")
 sample = pd.read_csv(f"{INPUT}sample_submission.csv" )
def crps(y_true, y_pred):
return K.mean(K.square(y_true - K.cumsum(y_pred, axis=1)) , axis=1 )<choose_model_class>
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]
def get_model() :
x = keras.layers.Input(shape=[X_train.shape[1]])
 fc1 = keras.layers.Dense(units=450, input_shape=[X_train.shape[1]] )(x)
 act1 = keras.layers.PReLU()(fc1)
 bn1 = keras.layers.BatchNormalization()(act1)
 dp1 = keras.layers.Dropout(0.55 )(bn1)
 gn1 = keras.layers.GaussianNoise(0.15 )(dp1)
 concat1 = keras.layers.Concatenate()([x, gn1])
 fc2 = keras.layers.Dense(units=600 )(concat1)
 act2 = keras.layers.PReLU()(fc2)
 bn2 = keras.layers.BatchNormalization()(act2)
 dp2 = keras.layers.Dropout(0.55 )(bn2)
 gn2 = keras.layers.GaussianNoise(0.15 )(dp2)
 concat2 = keras.layers.Concatenate()([concat1, gn2])
 fc3 = keras.layers.Dense(units=400 )(concat2)
 act3 = keras.layers.PReLU()(fc3)
 bn3 = keras.layers.BatchNormalization()(act3)
 dp3 = keras.layers.Dropout(0.55 )(bn3)
 gn3 = keras.layers.GaussianNoise(0.15 )(dp3)
 concat3 = keras.layers.Concatenate([concat2, gn3])
 output = keras.layers.Dense(units=199, activation='softmax' )(concat2)
 model = keras.models.Model(inputs=[x], outputs=[output])
 return model
def train_model(X_train, y_train, X_val, y_val):
model = get_model()
model.compile(optimizer=RAdam(warmup_proportion=0.1, min_lr=1e-7), loss=crps)
 er = EarlyStopping(patience=20, min_delta=1e-4, restore_best_weights=True, monitor='val_loss')
 model.fit(X_train, y_train, epochs=200, callbacks=[er], validation_data=[X_val, y_val], batch_size=batch_size)
 return model<split>
def prepare_data(X,metadata,weather,test=False,new_df=None):
if test :
X = X.drop(columns=['row_id'])
 add_lag_feature(weather, window=72)
 X = X.merge(metadata, on='building_id',how='left')
 X = X.merge(weather, on=['site_id', 'timestamp'], how='left')
 X['timestamp'] = pd.to_datetime(arg=X['timestamp'])
 X['year'] = X['timestamp'].dt.year
X['month'] = X['timestamp'].dt.month
X['day'] = X['timestamp'].dt.day
X['hour'] = X['timestamp'].dt.hour
X['weekday'] = X['timestamp'].dt.weekday
X = reduce_mem_usage(X)
 beaufort = [(0, 0, 0.3),(1, 0.3, 1.6),(2, 1.6, 3.4),(3, 3.4, 5.5),(4, 5.5, 8),(5, 8, 10.8),(6, 10.8, 13.9), 
(7, 13.9, 17.2),(8, 17.2, 20.8),(9, 20.8, 24.5),(10, 24.5, 28.5),(11, 28.5, 33),(12, 33, 200)]
for item in beaufort:
X.loc[(X['wind_speed']>=item[1])&(X['wind_speed']<item[2]), 'beaufort_scale'] = item[0]
X['age'] = X['year'] - X['year_built']
if new_df is None:
new_df = X.groupby(by=['building_id'], as_index=False)['timestamp'].min()
new_df = new_df.rename(columns = {'timestamp': 'start_ts'})
 X = X.merge(new_df, on = 'building_id', how='left')
 X['hours_passed'] =(X['timestamp'] - X['start_ts'] ).dt.total_seconds() /3600
X = reduce_mem_usage(X)
 if not test:
X = X.query('not(site_id==0 & timestamp<"2016-05-21 00:00:00")')
 cols = ['floor_count', 'air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed']
X.loc[:, cols] = X.loc[:, cols].interpolate(axis=0)
 cat_cols = ['meter', 'primary_use', 'site_id', 'building_id', 'year', 'month', 'day', 'hour']
for col in cat_cols:
X[col] = X[col].astype('category')
 X = reduce_mem_usage(X)
 if not test:
return X,new_df
else:
return X
rkf = RepeatedKFold(n_splits=5, n_repeats=5)
 models = []
for tr_idx, vl_idx in rkf.split(X_train, y_train):
x_tr, y_tr = X_train[tr_idx], y_train[tr_idx]
x_vl, y_vl = X_train[vl_idx], y_train[vl_idx]
model = train_model(x_tr, y_tr, x_vl, y_vl)
 models.append(model )<categorify>
df_train = reduce_mem_usage(df=df_train)
 weather_train = reduce_mem_usage(df=weather_train)
 bldg_metadata = reduce_mem_usage(df=bldg_metadata )
def make_pred(df, sample, env, models):
df['StadiumType'] = df['StadiumType'].apply(clean_StadiumType)
 df['StadiumType'] = df['StadiumType'].apply(transform_StadiumType)
 df['DefendersInTheBox_vs_Distance'] = df['DefendersInTheBox'] / df['Distance']
df['OffenseFormation'] = df['OffenseFormation'].apply(lambda x: x if x in off_form else np.nan)
 df = pd.concat([df.drop(['OffenseFormation'], axis=1), pd.get_dummies(df['OffenseFormation'], prefix='Formation')], axis=1)
 missing_cols = set(dummy_col)- set(df.columns)-set('Yards')
 for c in missing_cols:
df[c] = 0
df = df[dummy_col]
df.drop(['Yards'], axis=1, inplace=True)
 df['Turf'] = df['Turf'].map(Turf)
 df['Turf'] = df['Turf'] == 'Natural'
df['PossessionTeam'] = df['PossessionTeam'].map(map_abbr)
 df['HomeTeamAbbr'] = df['HomeTeamAbbr'].map(map_abbr)
 df['VisitorTeamAbbr'] = df['VisitorTeamAbbr'].map(map_abbr)
 df['HomePossesion'] = df['PossessionTeam'] == df['HomeTeamAbbr']
df['Field_eq_Possession'] = df['FieldPosition'] == df['PossessionTeam']
df['HomeField'] = df['FieldPosition'] == df['HomeTeamAbbr']
df['Formation_Heavy_Score'] = df["OffensePersonnel"].apply(formation_numeric)
 df['GameClock'] = df['GameClock'].apply(strtoseconds)
 df['PlayerHeight'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
df['PlayerBMI'] = 703*(df['PlayerWeight']/(df['PlayerHeight'])**2)
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(str_to_float)
 df['WindDirection'] = df['WindDirection'].apply(clean_WindDirection)
 df['WindDirection'] = df['WindDirection'].apply(transform_WindDirection)
 df['PlayDirection'] = df['PlayDirection'].apply(lambda x: x.strip() == 'right')
 df['Team'] = df['Team'].apply(lambda x: x.strip() =='home')
 indoor = "indoor"
df['GameWeather'] = df['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.lower().replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly' ).replace('clear and sunny', 'sunny and clear' ).replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather'] = df['GameWeather'].apply(map_weather)
 df['IsRusher'] = df['NflId'] == df['NflIdRusher']
df['X'] = df.apply(lambda row: row['X'] if row['PlayDirection'] else 120-row['X'], axis=1)
 df['Orientation'] = df.apply(lambda row: new_orientation(row['Orientation'], row['PlayDirection']), axis=1)
 df['Dir'] = df.apply(lambda row: new_orientation(row['Dir'], row['PlayDirection']), axis=1)
 df['YardsLeft'] = df.apply(lambda row: 100-row['YardLine'] if row['HomeField'] else row['YardLine'], axis=1)
 df['YardsLeft'] = df.apply(lambda row: row['YardsLeft'] if row['PlayDirection'] else 100-row['YardsLeft'], axis=1)
 df = df.sort_values(by=['PlayId', 'Team', 'IsRusher', 'JerseyNumber'] ).reset_index()
df = df.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate', 'NflId', 'NflIdRusher', 'GameId', 'PlayId', 'index', 'IsRusher', 'Team'], axis=1)
 cat_features = []
for col in df.columns:
if df[col].dtype =='object':
cat_features.append(col)
 df = df.drop(cat_features, axis=1)
 df.fillna(-999, inplace=True)
 X = np.array(df[players_col] ).reshape(-1, len(players_col)*22)
 play_col = df.drop(players_col, axis=1 ).columns
X_play_col = np.zeros(shape=(X.shape[0], len(play_col)))
 for i, col in enumerate(play_col):
X_play_col[:, i] = df[col][::22]
X = np.concatenate([X, X_play_col], axis=1)
 X = scaler.transform(X)
 y_pred = np.mean([np.cumsum(model.predict(X), axis=1)for model in models], axis=0)
 yardsleft = np.array(df['YardsLeft'][::22])
 for i in range(len(yardsleft)) :
y_pred[i, :yardsleft[i]-1] = 0
y_pred[i, yardsleft[i]+100:] = 1
env.predict(pd.DataFrame(data=y_pred.clip(0,1),columns=sample.columns))
return y_pred<predict_on_test>
df_train,new_df = prepare_data(df_train,bldg_metadata,weather_train)
 del weather_train;gc.collect()
for test, sample in tqdm.tqdm(env.iter_test()):
make_pred(test, sample, env, models )<save_to_csv>
df_train = reduce_mem_usage(df_train)
 df_train['group'] = df_train['month']
df_train['group'].replace(( 1,2,3,4,5,6), 1,inplace=True)
 df_train['group'].replace(( 7,8,9,10,11,12), 2, inplace=True)
 df_train['group'].value_counts()
env.write_submission_file()<import_modules>
numericals = ["square_feet", "year_built", "air_temperature", "cloud_coverage",
"dew_temperature", 'precip_depth_1_hr', 'floor_count', 'beaufort_scale','air_temperature_mean_lag72', 'air_temperature_max_lag72',
'air_temperature_min_lag72', 'air_temperature_std_lag72']
categoricals = ["site_id", "building_id", "primary_use", "hour", "weekday", "meter", "wind_direction"]
target = 'meter_reading'
feat_cols = categoricals + numericals
import numpy as np
import pandas as pd
import math
import matplotlib.pyplot as plt
import datetime
from kaggle.competitions import nflrush
import tqdm
import re
from string import punctuation
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.model_selection import train_test_split
import keras
import keras.backend as K<find_best_params>
df_train[target] = np.log1p(df_train[target] )
env = nflrush.make_env()<load_from_csv>
X_half_1 = df_train.loc[df_train.group==1][feat_cols]
X_half_2 = df_train.loc[df_train.group==2][feat_cols]
y_half_1 = df_train.loc[df_train.group==1][target]
y_half_2 = df_train.loc[df_train.group==2][target]
train = pd.read_csv(".. /input/nfl-big-data-bowl-2020/train.csv", dtype={'WindSpeed': 'object'} )<categorify>
d_half_1 = lgb.Dataset(X_half_1, label=y_half_1, categorical_feature=categoricals, free_raw_data=False)
 d_half_2 = lgb.Dataset(X_half_2, label=y_half_2, categorical_feature=categoricals, free_raw_data=False)
 watchlist_1 = [d_half_2, d_half_1]
watchlist_2 = [d_half_1, d_half_2]
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 45,
"learning_rate": 0.02,
"feature_fraction": 0.9,
"reg_lambda": 2,
"metric": "rmse"
}
class DataParser:
def __init__(self, data, predict=False, encoders={}):
self.data = data
self.predict = predict
self.encoders = encoders
def cleanWindSpeed(self, x):
x = str(x)
 x = x.lower()
if '-' in x:
x =(int(x.split('-')[0])+ int(x.split('-')[1])) / 2
elif ' gusts up to 25 ' in x:
x =(int(x.split(' gusts up tp 25 ')))
 try:
return float(x)
 except:
return -1
def cleanGameWeather(self, x):
x = str(x ).lower()
if 'sunny' in x or 'clear' in x or 'fair' in x:
return 'sunny'
elif 'cloud' in x or 'coudy' in x or 'clouidy' in x or 'hazy' in x or 'sun & clouds' in x or 'overcast' in x:
return 'cloudy'
elif 'rain' in x or 'shower' in x or 'rainy' in x:
return 'rainy'
elif 'controlled climate' in x or 'indoor' in x:
return 'indoor'
elif 'snow' in x:
return 'snowy'
return 'missing'
def mapGameWeather(self, txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def cleanStadiumType(self, txt):
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = re.sub(' +', ' ', txt)
 txt = txt.strip()
txt = txt.replace('outside', 'outdoor')
 txt = txt.replace('outdor', 'outdoor')
 txt = txt.replace('outddors', 'outdoor')
 txt = txt.replace('outdoors', 'outdoor')
 txt = txt.replace('oudoor', 'outdoor')
 txt = txt.replace('indoors', 'indoor')
 txt = txt.replace('ourdoor', 'outdoor')
 txt = txt.replace('retractable', 'rtr.')
 return txt
def cleanStadiumType2(self, txt):
if pd.isna(txt):
return np.nan
if 'outdoor' in txt or 'open' in txt:
return 1
if 'indoor' in txt or 'closed' in txt:
return 0
return np.nan
def cleanDefencePersonnel(self):
arr = [[int(s[0])for s in t.split(', ')]
for t in self.data['DefensePersonnel']]
self.data['DL'] = pd.Series([int(a[0])for a in arr])
 self.data['LB'] = pd.Series([int(a[1])for a in arr])
 self.data['DB'] = pd.Series([int(a[2])for a in arr])
 self.data = self.data.drop(labels=["DefensePersonnel"], axis=1)
 def cleanOffencePersonnel(self):
arr = [[int(s[0])for s in t.split(", ")]
for t in self.data["OffensePersonnel"]]
self.data["RB"] = pd.Series([int(a[0])for a in arr])
 self.data["TE"] = pd.Series([int(a[1])for a in arr])
 self.data["WR"] = pd.Series([int(a[2])for a in arr])
 self.data = self.data.drop(labels=["OffensePersonnel"], axis=1)
 def cleanOffenseFormation(self):
self.data['OffenseFormation'].fillna('missing', inplace=True)
 if(not self.predict):
le = LabelEncoder()
le.fit(self.data['OffenseFormation'])
 self.encoders['OffenseFormation'] = le
self.data['OffenseFormation'] = self.encoders['OffenseFormation'].transform(self.data['OffenseFormation'])
 def cleanHeight(self):

 self.data['PlayerHeight'] = self.data['PlayerHeight'].apply(
 lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
def cleanTimeHandoff(self):
self.data['TimeHandoff'] = self.data['TimeHandoff'].apply(
 lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
def cleanTimeSnap(self):
self.data['TimeSnap'] = self.data['TimeSnap'].apply(
 lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
def cleanGameClock(self):
arr = [[int(s[0])for s in t.split(":")]
for t in self.data["GameClock"]]
self.data["GameHour"] = [int(a[0])for a in arr]
self.data["GameMinute"] = [int(a[1])for a in arr]
self.data = self.data.drop(labels=['GameClock'], axis=1)
 def cleanTurf(self):
Turf = {'Field Turf': 'Artificial', 'A-Turf Titan': 'Artificial', 'Grass': 'Natural', 'UBU Sports Speed S5-M': 'Artificial',
'Artificial': 'Artificial', 'DD GrassMaster': 'Artificial', 'Natural Grass': 'Natural',
'UBU Speed Series-S5-M': 'Artificial', 'FieldTurf': 'Artificial', 'FieldTurf 360': 'Artificial', 'Natural grass': 'Natural', 'grass': 'Natural',
'Natural': 'Natural', 'Artifical': 'Artificial', 'FieldTurf360': 'Artificial', 'Naturall Grass': 'Natural', 'Field turf': 'Artificial',
'SISGrass': 'Artificial', 'Twenty-Four/Seven Turf': 'Artificial', 'natural grass': 'Natural'}
self.data['Turf'] = self.data['Turf'].map(Turf)
 self.data['Turf'] = self.data['Turf'] == 'Natural'
def cleanPossessionTeam(self):
map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in self.data['PossessionTeam'].unique() :
map_abbr[abb] = abb
self.data['PossessionTeam'] = self.data['PossessionTeam'].map(
 map_abbr)
 self.data['HomeTeamAbbr'] = self.data['HomeTeamAbbr'].map(map_abbr)
 self.data['VisitorTeamAbbr'] = self.data['VisitorTeamAbbr'].map(
 map_abbr)
 def cleanPlayerBirthDate(self):
self.data['PlayerBirthDate'] = self.data['PlayerBirthDate'].apply(
 lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
def cleanWindDirection(self, txt):
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = txt.replace('from', '')
 txt = txt.replace(' ', '')
 txt = txt.replace('north', 'n')
 txt = txt.replace('south', 's')
 txt = txt.replace('west', 'w')
 txt = txt.replace('east', 'e')
 return txt
def mapWindDirection(self, txt):
windDirectionMap = {
'n': 0,'nne': 1/8,'nen': 1/8,'ne': 2/8,
'ene': 3/8,'nee': 3/8,'e': 4/8,'ese': 5/8,
'see': 5/8,'se': 6/8,'ses': 7/8,'sse': 7/8,
's': 1,'ssw': 9/8,'sws': 9/8,'sw': 10/8,
'sww': 11/8,'wsw': 11/8,'w': 12/8,'wnw': 13/8,
'nw': 14/8,'nwn': 15/8,'nnw': 15/8
}
try:
return windDirectionMap[txt]
except:
return np.nan
def cleanPlayDirection(self):

 self.data['PlayDirection'] = self.data['PlayDirection'].apply(
 lambda x: x.strip() == 'right')
 def cleanTeam(self):

 self.data['Team'] = self.data['Team'].apply(
 lambda x: x.strip() == 'home')
 def isRusher(self):
self.data['isRusher'] = self.data['NflId'] == self.data['NflIdRusher']
temp = self.data[self.data['isRusher']][['Team', 'PlayId']].rename(columns={'Team':'RusherTeam'})
 self.data = self.data.merge(temp, on = 'PlayId')
 self.data['isRusherTeam'] = self.data['Team'] == self.data['RusherTeam']
self.data.drop(['NflId', 'NflIdRusher'], axis=1, inplace=True)
 def parse(self):
self.data['WindSpeed'] = self.data['WindSpeed'].apply(self.cleanWindSpeed)
 self.data['GameWeather'] = self.data['GameWeather'].apply(self.cleanGameWeather)
 self.data['GameWeather'] = self.data['GameWeather'].apply(self.mapGameWeather)
 self.data['StadiumType'] = self.data['StadiumType'].apply(self.cleanStadiumType)
 self.data['StadiumType'] = self.data['StadiumType'].apply(self.cleanStadiumType2)
 self.data['WindDirection'] = self.data['WindDirection'].apply(self.cleanWindDirection)
 self.data['WindDirection'] = self.data['WindDirection'].apply(self.mapWindDirection)
 self.cleanOffenseFormation()
self.cleanOffencePersonnel()
self.cleanDefencePersonnel()
self.cleanHeight()
self.cleanTimeHandoff()
self.cleanTimeSnap()
self.cleanTurf()
self.cleanPossessionTeam()
self.cleanPlayerBirthDate()
self.cleanPlayDirection()
self.cleanTeam()
self.isRusher()
if(not self.predict):
return self.data, self.encoders
return self.data<categorify>
print("Building model with first half and validating on second half:")
 model_half_1 = lgb.train(params, train_set=d_half_1, num_boost_round=10, valid_sets=watchlist_1, verbose_eval=200, early_stopping_rounds=200)
 print("Building model with second half and validating on first half:")
 model_half_2 = lgb.train(params, train_set=d_half_2, num_boost_round=10, valid_sets=watchlist_2, verbose_eval=200, early_stopping_rounds=200 )
parser = DataParser(train)
 train, encoders = parser.parse()<import_modules>
models = [model_half_1,model_half_2]
class FeatureEngine:
def __init__(self, data, exclude=[], deploy=False):
self.data = data
self.exclude = exclude
self.include = ['isRusher',
'HorizontalSandA',
'HomeField',
'FieldEqPossession',
'PlayerAge',
'HandSnapDelta',
'YardsLeft',
'BMI',
'DefendersInTheBox_vs_Distance']
self.deploy = deploy
def normalizeX(self, x_coordinate, play_direction):
if play_direction == 1:
return 120 - x_coordinate
else:
return x_coordinate
def normalizeOrientation(self, angle, play_direction):
if play_direction == 0:
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
def newLine(self, rush_team, field_position, yardline):
if rush_team == field_position:
return 10.0 + yardline
else:
return 60.0 +(50 - yardline)
 def euclideanDistance(self, x1, y1, x2, y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def backDirection(self, orientation):
if orientation > 180.0:
return 1
else:
return 0
def updateYardline(self, df):
new_yardline = df[df['isRusher']]
new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition','YardLine']].apply(lambda x: self.newLine(x[0],x[1],x[2]), axis=1)
 new_yardline = new_yardline[['GameId', 'PlayId', 'YardLine']]
return new_yardline
def updateOrientation(self, df, yardline):
df['X'] = df[['X','PlayDirection']].apply(lambda x: self.normalizeX(x[0],x[1]), axis=1)
 df['Orientation'] = df[['Orientation','PlayDirection']].apply(lambda x: self.normalizeOrientation(x[0],x[1]), axis=1)
 df = df.drop('YardLine', axis=1)
 df = pd.merge(df, yardline, on=['GameId', 'PlayId'], how='inner')
 return df
def backFeatures(self, df):
carriers = df[df['isRusher']][['GameId','PlayId','X','Y','Orientation','Dir','YardLine']]
carriers['back_from_scrimmage'] = carriers['YardLine'] - carriers['X']
carriers['back_oriented_down_field'] = carriers['Orientation'].apply(lambda x: self.backDirection(x))
carriers['back_moving_down_field'] = carriers['Dir'].apply(lambda x: self.backDirection(x))
carriers = carriers.rename(columns={'X':'back_X',
'Y':'back_Y'})
 carriers = carriers[['GameId','PlayId','back_X','back_Y', 'back_from_scrimmage', 'back_oriented_down_field', 'back_moving_down_field']]
return carriers
def featuresRelativeToBack(self, df, carriers):
player_distance = df[['GameId','PlayId','X','Y','isRusher']]
player_distance = pd.merge(player_distance, carriers, on=['GameId','PlayId'], how='inner')
 player_distance = player_distance[player_distance['isRusher'] == 0]
player_distance['dist_to_back'] = player_distance[['X','Y','back_X','back_Y']].apply(lambda x: self.euclideanDistance(x[0],x[1],x[2],x[3]), axis=1)
 player_distance = player_distance.groupby(['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field'])\ 
.agg({'dist_to_back':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field',
'min_dist','max_dist','mean_dist','std_dist']
return player_distance
def defenseFeatures(self, df):
rusher = df[df['isRusher']][['GameId','PlayId','Team','X','Y']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['isRusherTeam'] != True][['GameId','PlayId','X','Y','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: self.euclideanDistance(x[0],x[1],x[2],x[3]), axis=1)
 defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId','def_min_dist','def_max_dist','def_mean_dist','def_std_dist']
return defense
def rusherFeatures(self, df):
rusher = df[df['isRusher']][['GameId','PlayId','Dir', 'S', 'A', 'X', 'Y']]
rusher.columns = ['GameId','PlayId', 'RusherDir', 'RusherS', 'RusherA', 'RusherX', 'RusherY']
radian_angle =(90 - rusher['RusherDir'])* np.pi / 180.0
v_horizontal = np.abs(rusher['RusherS'] * np.cos(radian_angle))
v_vertical = np.abs(rusher['RusherS'] * np.sin(radian_angle))
rusher['v_horizontal'] = v_horizontal
rusher['v_vertical'] = v_vertical
rusher.columns = ['GameId','PlayId', 'RusherDir', 'RusherS','RusherA','RusherX', 'RusherY','v_horizontal', 'v_vertical']
return rusher
def engineerFieldEqPossession(self):
self.data['FieldEqPossession'] = self.data['FieldPosition'] == self.data['PossessionTeam']
def engineerHomeField(self):
self.data['HomeField'] = self.data['FieldPosition'] == self.data['HomeTeamAbbr']
def engineerHandoffSnapDelta(self):
self.data['TimeDelta'] = self.data.apply(lambda row:(
 row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 self.data = self.data.drop(['TimeHandoff', 'TimeSnap'], axis=1)
 def engineerYardsLeft(self):

 self.data['YardsLeft'] = self.data.apply(
 lambda row: 100-row['YardLine'] if row['HomeField'] else row['YardLine'], axis=1)
 self.data['YardsLeft'] = self.data.apply(
 lambda row: row['YardsLeft'] if row['PlayDirection'] else 100-row['YardsLeft'], axis=1)
 def engineerBMI(self):

 self.data['PlayerBMI'] = 703 * \ 
(self.data['PlayerWeight']/(self.data['PlayerHeight'])**2)
 def engineerPlayerAge(self):

 seconds_in_year = 60*60*24*365.25
self.data['PlayerAge'] = self.data.apply(lambda row:(
 row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 self.data = self.data.drop(['PlayerBirthDate'], axis=1)
 def engineerDefendersInTheBox_vs_Distance(self):
dfInBox_mode = self.data['DefendersInTheBox'].mode()
self.data['DefendersInTheBox'].fillna(
 dfInBox_mode.iloc[0], inplace=True)
 self.data['DefendersInTheBox_vs_Distance'] = self.data['DefendersInTheBox'] / \
self.data['Distance']
def combineFeatures(self, relative_to_back, defense,rushing, static):
df = pd.merge(relative_to_back,defense,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,rushing,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,static,on=['GameId','PlayId'],how='inner')
 return df
def engineer(self):
yardline = self.updateYardline(self.data)
 self.data = self.updateOrientation(self.data, yardline)
 back_feats = self.backFeatures(self.data)
 rel_back = self.featuresRelativeToBack(self.data, back_feats)
 def_feats = self.defenseFeatures(self.data)
 rush_feats = self.rusherFeatures(self.data)
 for c in self.include:
if c in self.exclude:
continue
elif c == 'FieldEqPossession':
self.engineerFieldEqPossession()
elif c == 'HomeField':
self.engineerHomeField()
elif c == 'YardsLeft':
self.engineerYardsLeft()
elif c == 'PlayerAge':
self.engineerPlayerAge()
elif c == 'HandSnapDelta':
self.engineerHandoffSnapDelta()
elif c == 'BMI':
self.engineerBMI()
elif c == 'DefendersInTheBox_vs_Distance':
self.engineerDefendersInTheBox_vs_Distance()
self.data = self.combineFeatures(rel_back, def_feats, rush_feats, self.data)
 self.data = self.data.drop_duplicates()
return self.data<categorify>
del X_half_1,X_half_2,d_half_1,d_half_2,df_train, watchlist_1,watchlist_2,y_half_1,y_half_2
gc.collect()
def DataReshaper2(data, predict=False, playersCol = []):

 data = data.sort_values(by=['PlayId', 'Team', 'isRusher', 'JerseyNumber'] ).reset_index()
data.drop(['GameId', 'PlayId', 'index', 'isRusher', 'Team'], axis=1, inplace=True)
 drop_col = []
for c in data.columns:
if data[c].dtype == 'object':
drop_col.append(c)
 data.drop(drop_col, axis=1, inplace=True)
 data.fillna(-999, inplace=True)
 players_col = playersCol
if(not predict):
for col in data.columns:
if data[col][:22].std() != 0:
players_col.append(col)
 X_train = np.array(data[players_col] ).reshape(-1, len(players_col)*22)
 if(not predict):
play_col = data.drop(players_col + ['Yards'], axis=1 ).columns
else:
play_col = data.drop(players_col, axis=1 ).columns
X_play_col = np.zeros(shape=(X_train.shape[0], len(play_col)))
 for i, col in enumerate(play_col):
X_play_col[:, i] = data[col][::22]
X_train = np.concatenate([X_train, X_play_col], axis=1)
 if(not predict):
y_train = np.zeros(shape=(X_train.shape[0], 199))
for i, yard in enumerate(train['Yards'][::22]):
y_train[i, yard+99:] = np.ones(shape=(1, 100-yard))
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 if(not predict):
return X_train, y_train, players_col
return X_train<prepare_x_and_y>
X_train, y_train, players_col = DataReshaper2(train )<import_modules>
df_test = pd.read_csv(f"{INPUT}test.csv")
 weather_test = pd.read_csv(f"{INPUT}weather_test.csv")
 df_test = reduce_mem_usage(df=df_test)
 weather_test = reduce_mem_usage(df=weather_test )
from keras.layers import Dense,Input,Dropout,BatchNormalization,LeakyReLU,PReLU,GaussianNoise
from keras.models import Model
import keras.backend as K
from keras.callbacks import EarlyStopping,ModelCheckpoint
from keras.optimizers import Adam
from keras.losses import binary_crossentropy
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, Callback<define_search_model>
df_test = prepare_data(df_test,bldg_metadata,weather_test,test=True,new_df=new_df )
def build_model() :
input_tensor = Input(shape=(X_train.shape[1],))
x = Dense(1024, activation='relu' )(input_tensor)
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 x = GaussianNoise(0.15 )(x)
 x = Dense(512, activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 x = GaussianNoise(0.15 )(x)
 x = Dense(256, activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.5 )(x)
 x = GaussianNoise(0.15 )(x)
 output_tensor = Dense(199, activation='softmax' )(x)
 model = Model(input_tensor, output_tensor)
 model.compile(optimizer='adam', loss=crps_loss, metrics=[])
 return model<choose_model_class>
df_test = reduce_mem_usage(df_test )
ES = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
patience=3
)<train_model>
df_test = df_test.drop(columns=['start_ts','timestamp'])
class CRPSCallback(Callback):
def __init__(self, validation, predict_batch_size=1024, include_on_batch=False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
def on_batch_begin(self, batch, logs={}):
pass
def on_train_begin(self, logs={}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs={}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs={}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s<compute_test_metric>
del bldg_metadata,weather_test,new_df;gc.collect()
def crps_score(y_prediction, y_valid, shape=X_train.shape[0]):
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_prediction, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * shape)
 crps = np.round(val_s, 6)
 return crps<compute_test_metric>
test = df_test[feat_cols]
def crps_loss(y_true, y_pred):
return K.mean(K.square(y_true - K.cumsum(y_pred, axis=1)) , axis=1 )<train_model>
i=0
res=[]
step_size = 50000
for j in tqdm(range(int(np.ceil(test.shape[0]/50000)))) :
res.append(sum(np.expm1([model.predict(test.iloc[i:i+step_size])for model in models])/len(models)))
 i+=step_size
EPOCHS = 200
BATCH_SIZE = 1024
def train_model(X_train, y_train, X_val, y_val):
model = build_model()
model.fit(X_train, y_train,
validation_data=(X_val, y_val),
epochs=EPOCHS,
batch_size=BATCH_SIZE,
callbacks=[CRPSCallback(validation =(X_val, y_val)) , ES],
verbose=1)
 y_pred = model.predict(X_val)
 y_valid = y_val
crps = crps_score(y_pred, y_valid, shape=X_val.shape[0])
 return model, crps<split>
res = np.concatenate(res )
N_SPLITS = 3
N_REPEATS = 2
nn_crps = []
models = []
for n in range(N_REPEATS):
kf = KFold(N_SPLITS, random_state= 21 + n, shuffle = True)
 for k_fold,(tr_idx, vl_idx)in enumerate(kf.split(y_train)) :
print("-----------")
 print(f'Loop {n+1}/{N_REPEATS}' + f' Fold {k_fold+1}/{N_SPLITS}')
 print("-----------")
 x_tr, y_tr = X_train[tr_idx], y_train[tr_idx]
x_vl, y_vl = X_train[vl_idx], y_train[vl_idx]
model, crps = train_model(x_tr, y_tr, x_vl, y_vl)
 models.append(model)
 print("the %d fold crps(NN)is %f"%(( k_fold+1), crps))
nn_crps.append(crps )<predict_on_test>
submission = pd.read_csv('/kaggle/input/ashrae-energy-prediction/sample_submission.csv')
 submission['meter_reading'] = res
submission.loc[submission['meter_reading']<0, 'meter_reading'] = 0
submission.to_csv('submission.csv', index=False, float_format='%.4f')
def make_pred(df, sample, env, models):
parser = DataParser(df, predict=True, encoders=encoders)
 df = parser.parse()
engine = FeatureEngine(df)
 df = engine.engineer()
X = DataReshaper2(df, predict=True, playersCol=players_col)
 y_pred = np.mean([np.cumsum(model.predict(X), axis=1)for model in models], axis=0)
 yardsleft = np.array(df['YardsLeft'][::22])
 for i in range(len(yardsleft)) :
y_pred[i, :int(yardsleft[i]-1)] = 0
y_pred[i, int(yardsleft[i]+100):] = 1
env.predict(pd.DataFrame(data=y_pred.clip(0,1),columns=sample.columns))
return y_pred<predict_on_test>
%%time
leak_score0 = 0
leak_df = pd.read_pickle('/kaggle/input/ashrae-ucf-spider-and-eda-full-test-labels/site0.pkl')
 leak_df['meter_reading'] = leak_df.meter_reading_scraped
leak_df.drop(['meter_reading_original','meter_reading_scraped'], axis=1, inplace=True)
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[leak_df.timestamp.dt.year > 2016]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
submission.loc[submission.meter_reading < 0, 'meter_reading'] = 0
for bid in leak_df.building_id.unique() :
temp_df = leak_df[(leak_df.building_id == bid)]
for m in temp_df.meter.unique() :
v0 = submission.loc[(df_test.building_id == bid)&(df_test.meter==m), 'meter_reading'].values
v1 = temp_df[temp_df.meter==m].meter_reading.values
leak_score0 += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)
 submission.loc[(df_test.building_id == bid)&(df_test.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values
leak_score0 /= len(leak_df )
for test, sample in tqdm.tqdm(env.iter_test()):
make_pred(test, sample, env, models )<save_to_csv>
%%time
leak_score1 = 0
leak_df = pd.read_pickle('/kaggle/usr/lib/ucl_data_leakage_episode_2/site1.pkl')
 leak_df['meter_reading'] = leak_df.meter_reading_scraped
leak_df.drop(['meter_reading_scraped'], axis=1, inplace=True)
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[leak_df.timestamp.dt.year > 2016]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
for bid in leak_df.building_id.unique() :
temp_df = leak_df[(leak_df.building_id == bid)]
for m in temp_df.meter.unique() :
v0 = submission.loc[(df_test.building_id == bid)&(df_test.meter==m), 'meter_reading'].values
v1 = temp_df[temp_df.meter==m].meter_reading.values
leak_score1 += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)
 submission.loc[(df_test.building_id == bid)&(df_test.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values
leak_score1 /= len(leak_df )
env.write_submission_file()<set_options>
submission.to_csv('submission_ucf_replaced.csv', index=False, float_format='%.4f' )