Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<save_to_csv>
path_data = "/kaggle/input/ashrae-energy-prediction/"
path_train = path_data + "train.csv"
path_test = path_data + "test.csv"
path_building = path_data + "building_metadata.csv"
path_weather_train = path_data + "weather_train.csv"
path_weather_test = path_data + "weather_test.csv"
path_drops = path_data + ".. /lier-list/rows_to_drop.csv"
path_model = path_data + ".. /building-models/building_meter_models.pkl"
plt.style.use("seaborn")
 sns.set(font_scale=1)
 myfavouritenumber = 0
seed = myfavouritenumber
random.seed(seed )
sub_df.to_csv("submission.csv", index=False )<define_variables>
df_train = pd.read_csv(path_train)
 building = pd.read_csv(path_building)
 le = LabelEncoder()
building.primary_use = le.fit_transform(building.primary_use)
 weather_train = pd.read_csv(path_weather_train)
 bad_rows = pd.read_csv(path_drops)
 df_train = df_train.drop(index=bad_rows["0"] )
notebookstart= time.time()
random.seed(2018)
 get_ipython().magic('matplotlib inline')
 id_col = "ID"
target_var = "target"
Debug = False
Home = False
Build_Results_csv = False
results = pd.DataFrame(columns = ["Rounds","Score","STDV", "LB", "Parameters"])
 if Build_Results_csv is True & Home is True: results.to_csv("results.csv")
 if Home is True:
path = r"D:\My Computer\DATA\Santander"
os.chdir(path)
 print("Data Load Stage")
 training = pd.read_csv('train.csv', index_col = id_col)
 if Debug is True : training = training.sample(100)
 traindex = training.index
testing = pd.read_csv('test.csv', index_col = id_col)
 if Debug is True : testing = testing.sample(100)
 testdex = testing.index
else:
print("Data Load Stage")
 training = pd.read_csv('.. /input/train.csv', index_col = id_col)
 if Debug is True : training = training.sample(100)
 traindex = training.index
testing = pd.read_csv('.. /input/test.csv', index_col = id_col)
 if Debug is True : testing = testing.sample(100)
 testdex = testing.index
y = np.log1p(training[target_var])
 training.drop(target_var,axis=1, inplace=True)
 print('Train shape: {} Rows, {} Columns'.format(*training.shape))
print('Test shape: {} Rows, {} Columns'.format(*testing.shape))
print("Combine Train and Test")
 df = pd.concat([training,testing],axis=0)
 del training, testing
gc.collect()
print('
All Data shape: {} Rows, {} Columns'.format(*df.shape))<train_model>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df
test_df = df.loc[testdex,:]
vocab = df.columns
lgtrain = lgb.Dataset(df.loc[traindex,vocab],y ,feature_name = "auto")
 print("Starting LightGBM.Train shape: {}, Test shape: {}".format(df.loc[testdex,:].shape,test_df.shape))
print("Feature Num: ",len(vocab))
del df; gc.collect() ;<init_hyperparams>
df_train = reduce_mem_usage(df_train, use_float16=True)
 building = reduce_mem_usage(building, use_float16=True)
 weather_train = reduce_mem_usage(weather_train, use_float16=True )
print("Light Gradient Boosting Regressor: ")
 lgbm_params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
"learning_rate": 0.01,
"num_leaves": 180,
"feature_fraction": 0.50,
"bagging_fraction": 0.50,
'bagging_freq': 4,
"max_depth": -1,
"reg_alpha": 0.3,
"reg_lambda": 0.1,
"min_child_weight":10,
'zero_as_missing':True
}<find_best_params>
def prepare_data(X, building_data, weather_data, test=False):

 X = X.merge(building_data, on="building_id", how="left")
 X = X.merge(weather_data, on=["site_id", "timestamp"], how="left")
 X.timestamp = pd.to_datetime(X.timestamp, format="%Y-%m-%d %H:%M:%S")
 if not test:
X.sort_values("timestamp", inplace=True)
 X.reset_index(drop=True, inplace=True)
 gc.collect()
holidays = ["2016-01-01", "2016-01-18", "2016-02-15", "2016-05-30", "2016-07-04",
"2016-09-05", "2016-10-10", "2016-11-11", "2016-11-24", "2016-12-26",
"2017-01-01", "2017-01-16", "2017-02-20", "2017-05-29", "2017-07-04",
"2017-09-04", "2017-10-09", "2017-11-10", "2017-11-23", "2017-12-25",
"2018-01-01", "2018-01-15", "2018-02-19", "2018-05-28", "2018-07-04",
"2018-09-03", "2018-10-08", "2018-11-12", "2018-11-22", "2018-12-25",
"2019-01-01"]
X["hour"] = X.timestamp.dt.hour
X["weekday"] = X.timestamp.dt.weekday
X["is_holiday"] =(X.timestamp.dt.date.astype("str" ).isin(holidays)).astype(int)
 drop_features = ["timestamp", "sea_level_pressure", "wind_direction", "wind_speed",
"site_id", "primary_use", "square_feet", "year_built", "floor_count"]
X.drop(drop_features, axis=1, inplace=True)
 return X
modelstart= time.time()
lgb_cv = lgb.cv(
 params = lgbm_params,
train_set = lgtrain,
num_boost_round=2000,
stratified=False,
nfold = 5,
verbose_eval=50,
seed = 23,
early_stopping_rounds=75)
 optimal_rounds = np.argmin(lgb_cv['rmse-mean'])
 best_cv_score = min(lgb_cv['rmse-mean'])
 print("
Optimal Round: {}
Optimal Score: {} + {}".format(
 optimal_rounds,best_cv_score,lgb_cv['rmse-stdv'][optimal_rounds]))
results = results.append({"Rounds": optimal_rounds,
"Score": best_cv_score,
"STDV": lgb_cv['rmse-stdv'][optimal_rounds],
"LB": None,
"Parameters": lgbm_params}, ignore_index=True)
 if Home is True:
with open('results.csv', 'a')as f:
results.to_csv(f, header=False )<sort_values>
X_train = prepare_data(df_train, building, weather_train )
pd.set_option('max_colwidth', 800)
 display(results.sort_values(by="Score",ascending = True))<find_best_params>
del df_train
gc.collect()
learning_rates = [0.012,0.008]
for param in learning_rates:
print("Learning Rate: ", param)
 modelstart= time.time()
lgbm_params["learning_rate"] = param
lgb_cv = lgb.cv(
 params = lgbm_params,
train_set = lgtrain,
num_boost_round=10000,
stratified=False,
nfold = 5,
verbose_eval=200,
seed = 23,
early_stopping_rounds=75)
 optimal_rounds = np.argmin(lgb_cv['rmse-mean'])
 best_cv_score = min(lgb_cv['rmse-mean'])
 print("Optimal Round: {}
Optimal Score: {} + {}".format(
 optimal_rounds,best_cv_score,lgb_cv['rmse-stdv'][optimal_rounds]))
print("
results = results.append({"Rounds": optimal_rounds,
"Score": best_cv_score,
"STDV": lgb_cv['rmse-stdv'][optimal_rounds],
"LB": None,
"Parameters": lgbm_params}, ignore_index=True)
 if Home is True:
with open('results.csv', 'a')as f:
results.to_csv(f, header=False)
<sort_values>
def LGBM(X_train, y_train):
X_half_1 = X_train[:int(X_train.shape[0] / 2)]
X_half_2 = X_train[int(X_train.shape[0] / 2):]
y_half_1 = y_train[:int(X_train.shape[0] / 2)]
y_half_2 = y_train[int(X_train.shape[0] / 2):]
categorical_features = ["hour", "weekday"]
d_half_1 = lgb.Dataset(X_half_1, label=y_half_1, categorical_feature=categorical_features, free_raw_data=False)
 d_half_2 = lgb.Dataset(X_half_2, label=y_half_2, categorical_feature=categorical_features, free_raw_data=False)
 watchlist_1 = [d_half_1, d_half_2]
watchlist_2 = [d_half_2, d_half_1]
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 40,
"learning_rate": 0.05,
"feature_fraction": 0.85,
"reg_lambda": 2,
"metric": "rmse"
}
model_half_1 = lgb.train(params, train_set=d_half_1, num_boost_round=1000, valid_sets=watchlist_1, verbose_eval=200, early_stopping_rounds=200)
 model_half_2 = lgb.train(params, train_set=d_half_2, num_boost_round=1000, valid_sets=watchlist_2, verbose_eval=200, early_stopping_rounds=200)
 return model_half_1, model_half_2
pd.set_option('max_colwidth', 800)
 display(results.sort_values(by="Score",ascending = True))<find_best_params>
def modeling(X_train, stop=999999):
drop_list = ["building_id", "meter", "meter_reading"]
ids = np.sort(X_train["building_id"].unique())
 df_model = pd.DataFrame(columns=["building_id", "meter", "half_1", "half_2"])
 index = 0
for bid in ids:
if stop < bid:
break
meters = np.sort(X_train[X_train["building_id"] == bid]["meter"].unique())
 for meter in meters:
Modeling building_id:{}, meter:{}
---------
".format(bid, meter))
X_each = X_train[(X_train["building_id"] == bid)&(X_train["meter"] == meter)]
y_each = np.log1p(X_each.meter_reading)
 X_each.drop(drop_list, axis=1, inplace=True)
 model_half_1, model_half_2 = LGBM(X_each, y_each)
 df_model.loc[index] = [bid, meter, model_half_1, model_half_2]
index+=1
return df_model
final_model_params = results.iloc[results["Score"].idxmin() ,:]["Parameters"]
optimal_rounds = results.iloc[results["Score"].idxmin() ,:]["Rounds"]
print("Parameters for Final Models:
",final_model_params)
 print("Score: {} +/- {}".format(results.iloc[results["Score"].idxmin() ,:]["Score"],results.iloc[results["Score"].idxmin() ,:]["STDV"]))
print("Rounds: ", optimal_rounds )<train_model>
df_model = pd.read_pickle(path_model)
 df_model
allmodelstart= time.time()
multi_seed_pred = dict()
all_feature_importance_df = pd.DataFrame()
all_seeds = [27,22,300,401]
for seeds_x in all_seeds:
modelstart= time.time()
print("Seed: ", seeds_x,)
 final_model_params["seed"] = seeds_x
lgb_reg = lgb.train(
 final_model_params,
lgtrain,
num_boost_round = optimal_rounds + 1,
verbose_eval=200)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = vocab
fold_importance_df["importance"] = lgb_reg.feature_importance()
all_feature_importance_df = pd.concat([all_feature_importance_df, fold_importance_df], axis=0)
 multi_seed_pred[seeds_x] = list(lgb_reg.predict(test_df))
print("Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))
print("
del lgb_reg
cols = all_feature_importance_df[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = all_feature_importance_df.loc[all_feature_importance_df.feature.isin(cols)]
plt.figure(figsize=(8,10))
sns.barplot(x="importance", y="feature",
data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features(avg over folds)')
 plt.tight_layout()
plt.savefig('lgbm_importances.png')
 print("All Model Runtime: %0.2f Minutes"%(( time.time() - allmodelstart)/60))
sub_preds = pd.DataFrame.from_dict(multi_seed_pred ).replace(0,0.000001)
 del multi_seed_pred; gc.collect() ;
f, ax = plt.subplots(figsize=[8,6])
 sns.heatmap(sub_preds.corr() ,
annot=True, fmt=".2f",cbar_kws={'label': 'Percentage %'},cmap="plasma",ax=ax)
 ax.set_title("Correlation Plot for Seed Diversified Models")
 plt.show()<save_to_csv>
drop_list = ["building_id", "meter", "meter_reading"]
model_names = ["half_1", "half_2"]
colmuns = X_train[(X_train["building_id"] == 0)&(X_train["meter"] == 0)].drop(drop_list, axis=1 ).columns.values
for bid in [31, 15, 75]:
meters = np.sort(X_train[X_train["building_id"] == bid]["meter"].unique())
 for meter in meters:
models = []
for m_name in model_names:
model = df_model[(df_model["building_id"] == bid)&(df_model["meter"] == meter)][m_name].values[0]
models.append(model)
 feature_importance(colmuns, models, bid, meter )
mean_sub = np.expm1(sub_preds.mean(axis=1 ).rename(target_var))
mean_sub.index = testdex
mean_sub.to_csv('mean_sub_ep{}_sc{}.csv'.format(optimal_rounds,round(best_cv_score,5))
,index = True, header=True)
 print("Notebook Runtime: %0.2f Minutes"%(( time.time() - notebookstart)/60))
mean_sub.head()<set_options>
del X_train
gc.collect()
print('loading libs...')
 warnings.filterwarnings("ignore")
 print('done' )<set_options>
df_test = pd.read_csv(path_test)
 weather_test = pd.read_csv(path_weather_train)
 df_test = reduce_mem_usage(df_test)
 weather_test = reduce_mem_usage(weather_test)
 X_test = prepare_data(df_test, building, weather_test, test=True )
env = nflrush.make_env()<define_variables>
del df_test, building
gc.collect()
DF_NAME= []
PATH = '.. /input/nfl-big-data-bowl-2020/'
SEED = 1229
NFOLDS = 5
NREPEATS = 5<load_from_csv>
def predicting(X_test, df_model, stop=9999999):
drop_list = ["building_id", "meter", "row_id"]
ids = np.sort(X_test["building_id"].unique())
 submission = pd.DataFrame(columns=["row_id", "meter_reading"])
 for bid in ids:
if stop < bid:
break
meters = np.sort(X_test[X_test["building_id"] == bid]["meter"].unique())
 for meter in meters:
Predecting building_id:{}, meter:{}
---------
".format(bid, meter))
X_each = X_test[(X_test["building_id"] == bid)&(X_test["meter"] == meter)]
row_ids = X_each.row_id
X_each.drop(drop_list, axis=1, inplace=True)
 model_half_1 = df_model[(df_model["building_id"] == bid)&(df_model["meter"] == meter)]["half_1"].values[0]
model_half_2 = df_model[(df_model["building_id"] == bid)&(df_model["meter"] == meter)]["half_2"].values[0]
pred = np.expm1(model_half_1.predict(X_each, num_iteration=model_half_1.best_iteration)) / 2
pred += np.expm1(model_half_2.predict(X_each, num_iteration=model_half_2.best_iteration)) / 2
sub_each = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(pred, 0, a_max=None)})
 submission = pd.concat([submission, sub_each])
 return submission.sort_values("row_id" )
def SeedEverything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 return
def DataLoading(path, df_name):
files = os.listdir(f'{path}')
 for i in range(len(files)) :
s0 = files[i]
s1 = files[i][:-4]
s2 = files[i][-4:]
if s2 =='.csv':
print('loading:'+ s1 + '...')
 globals() [s1] = pd.read_csv(f'{path}'+ s0, dtype={'WindSpeed': 'object'})
 df_name.append(s1)
 elif s2 == '.pkl':
print('loading:'+ s1 + '...')
 globals() [s1] = pd.read_pickle(f'{path}'+ s0, dtype={'WindSpeed': 'object'})
 df_name.append(s1)
 else:
pass
print('successfully loading: ')
 print(df_name)
 print('done')
 return df_name
def DataStatistics(df):
summary = pd.DataFrame(df.dtypes,columns=['dtypes'])
 summary = summary.reset_index()
summary['Name'] = summary['index']
summary = summary[['Name','dtypes']]
summary['Missing'] = df.isnull().sum().values
summary['Missing_percentage'] = round(( summary['Missing']/df.shape[0])*100, 1)
 summary['Uniques'] = df.nunique().values
summary['First Value'] = df.loc[0].values
summary['Second Value'] = df.loc[1].values
summary['Third Value'] = df.loc[2].values
for name in summary['Name'].value_counts().index:
summary.loc[summary['Name'] == name, 'Entropy'] = round(stats.entropy(df[name].value_counts(normalize=True), base=2),2)
 summary.set_index('Name',inplace=True)
 summary = summary.T
return summary
def DataShowing(df_name, start=0, end=49, seeall = True):
if seeall:
pd.set_option('display.max_rows', None)
 pd.set_option('display.max_columns', None)
 df_name.sort(reverse=True)
 for i in range(len(df_name)) :
s = df_name[i]
df = globals() [s]
print('data shape of ' + s + ':' + f'{df.shape}')
 df = df.iloc[:,start:end]
if df.empty:
pass
else:
print('looking over the statistics of all features of ' + s + '...')
 display(DataStatistics(df))
print('looking over the statistics of the num_type_features of ' + s + '...')
 display(df.describe())
 return
def DataProcessing(train):
train['PossessionTeam'] = train['PossessionTeam'].map(map_abbr)
 train['HomeTeamAbbr'] = train['HomeTeamAbbr'].map(map_abbr)
 train['VisitorTeamAbbr'] = train['VisitorTeamAbbr'].map(map_abbr)
 train['HomePossesion'] = train['PossessionTeam'] == train['HomeTeamAbbr']
train = pd.concat([train.drop(['OffenseFormation'], axis=1),
pd.get_dummies(train['OffenseFormation'], prefix='Formation')], axis=1)
 globals() ['dummy_col'] = train.columns
train['GameClock'] = train['GameClock'].apply(strtoseconds)
 train['PlayerHeight'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 train['WindSpeed'] = train['WindSpeed'].apply(str_to_float)
 train['PlayDirection'] = train['PlayDirection'].apply(lambda x: x is 'right')
 train['Team'] = train['Team'].apply(lambda x: x.strip() =='home')
 train['GameWeather'] = train['GameWeather'].str.lower()
indoor = "indoor"
train['GameWeather'] = train['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(map_weather)
 train['IsRusher'] = train['NflId'] == train['NflIdRusher']
train.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate', 'WindDirection','NflId', 'NflIdRusher'],
axis=1, inplace=True)
 train = train.sort_values(by=['PlayId', 'Team', 'IsRusher'] ).reset_index()
train.drop(['GameId', 'PlayId', 'index', 'IsRusher', 'Team'], axis=1, inplace=True)
 cat_features = []
for col in train.columns:
if train[col].dtype =='object':
cat_features.append(col)
 train.drop(cat_features, axis=1,inplace=True)
 train.fillna(-999, inplace=True)
 globals() ['players_col'] = []
for col in train.columns:
if train[col][:22].std() !=0:
players_col.append(col)
 X_train = np.array(train[players_col] ).reshape(-1, 11*22)
 play_col = train.drop(players_col+['Yards'], axis=1 ).columns
X_play_col = np.zeros(shape=(X_train.shape[0], len(play_col)))
 for i, col in enumerate(play_col):
X_play_col[:, i] = train[col][::22]
X_train = np.concatenate([X_train, X_play_col], axis=1)
 y_train = np.zeros(shape=(X_train.shape[0], 199))
for i,yard in enumerate(train['Yards'][::22]):
y_train[i, yard+99:] = np.ones(shape=(1, 100-yard))
return X_train, y_train
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
def str_to_float(txt):
try:
return float(txt)
 except:
return -1
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def crps(y_true, y_pred):
ans = 0
ground_t = y_true.argmax(1)
 for i, t in enumerate(ground_t):
for n in range(-99, 100):
h = n>=(t-99)
 ans+=(y_pred[i][n+99]-h)**2
return ans/(199*len(y_true))
__all__ = ['RAdam']
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
learning_rate = kwargs.pop('lr', learning_rate)
 super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.learning_rate = K.variable(learning_rate, name='learning_rate')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(min_lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = K.maximum(self.total_steps - warmup_steps, 1)
 decay_rate =(self.min_lr - lr)/ decay_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr + decay_rate * K.minimum(t - warmup_steps, decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t))
self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t))
r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t >= 5, r_t * m_corr_t /(v_corr_t + self.epsilon), m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
@property
def lr(self):
return self.learning_rate
@lr.setter
def lr(self, learning_rate):
self.learning_rate = learning_rate
def get_config(self):
config = {
'learning_rate': float(K.get_value(self.learning_rate)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def ModelBuild(X_train, y_train):
score = 0
i = 0
folds = RepeatedKFold(n_splits=NFOLDS, n_repeats=NREPEATS, random_state = SEED)
 splits = folds.split(X_train,y_train)
 for fold_n,(train_index, valid_index)in enumerate(splits):
X_tr, X_val = X_train[train_index], X_train[valid_index]
y_tr, y_val = y_train[train_index], y_train[valid_index]
globals() ['model' + str(i)] = keras.models.Sequential([
keras.layers.Dense(units=512, input_shape=[X_tr.shape[1]], activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.3),
keras.layers.Dense(units=256, activation='relu'),
keras.layers.Dropout(0.2),
keras.layers.Dense(units=256, activation='relu'),
keras.layers.Dense(units=199, activation='sigmoid')
 ])
 globals() ['model' + str(i)].compile(optimizer=RAdam(warmup_proportion=0.1, min_lr=1e-5), loss='mse')
 es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=15,verbose=1, mode='auto', restore_best_weights=True)
 rlr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1,patience=10, min_lr=1e-6, mode='auto', verbose=1)
 globals() ['model' + str(i)].fit(X_tr,[y_tr], validation_data=(X_val,[y_val]), callbacks=[es, rlr],epochs=100, batch_size=128, verbose=0)
 cv_predict= globals() ['model' + str(i)].predict(X_val)
 crps_res = crps(y_val,cv_predict)
 print(f'The CV score is: {crps_res}')
 score +=crps_res
i += 1
score = score/(NFOLDS*NREPEATS)
 print(f'The averge CV score is: {score}')
 return
def make_pred(df, sample, env, model):
df['OffenseFormation'] = df['OffenseFormation'].apply(lambda x: x if x in off_form else np.nan)
 df = pd.concat([df.drop(['OffenseFormation'], axis=1), pd.get_dummies(df['OffenseFormation'], prefix='Formation')], axis=1)
 missing_cols = set(dummy_col)- set(test.columns)-set('Yards')
 for c in missing_cols:
df[c] = 0
df = df[dummy_col]
df.drop(['Yards'], axis=1, inplace=True)
 df['PossessionTeam'] = df['PossessionTeam'].map(map_abbr)
 df['HomeTeamAbbr'] = df['HomeTeamAbbr'].map(map_abbr)
 df['VisitorTeamAbbr'] = df['VisitorTeamAbbr'].map(map_abbr)
 df['HomePossesion'] = df['PossessionTeam'] == df['HomeTeamAbbr']
df['GameClock'] = df['GameClock'].apply(strtoseconds)
 df['PlayerHeight'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(str_to_float)
 df['PlayDirection'] = train['PlayDirection'].apply(lambda x: x is 'right')
 df['Team'] = df['Team'].apply(lambda x: x.strip() =='home')
 indoor = "indoor"
df['GameWeather'] = df['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.lower().replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly' ).replace('clear and sunny', 'sunny and clear' ).replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather'] = df['GameWeather'].apply(map_weather)
 df['IsRusher'] = df['NflId'] == df['NflIdRusher']
df = df.sort_values(by=['PlayId', 'Team', 'IsRusher'] ).reset_index()
df = df.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate', 'WindDirection', 'NflId', 'NflIdRusher', 'GameId', 'PlayId', 'index', 'IsRusher', 'Team'], axis=1)
 cat_features = []
for col in df.columns:
if df[col].dtype =='object':
cat_features.append(col)
 df = df.drop(cat_features, axis=1)
 df.fillna(-999, inplace=True)
 X = np.array(df[players_col] ).reshape(-1, 11*22)
 play_col = df.drop(players_col, axis=1 ).columns
X_play_col = np.zeros(shape=(X.shape[0], len(play_col)))
 for i, col in enumerate(play_col):
X_play_col[:, i] = df[col][::22]
X = np.concatenate([X, X_play_col], axis=1)
 y_pred = np.zeros(199 ).reshape(1,199)
 for i in range(NFOLDS*NREPEATS):
globals() ['y_pred' + str(i)] = globals() ['model' + str(i)].predict(X)
 y_pred += globals() ['y_pred' + str(i)]
y_pred = y_pred/(NFOLDS*NREPEATS)
 for pred in y_pred:
prev = 0
for i in range(len(pred)) :
if pred[i]<prev:
pred[i]=prev
prev=pred[i]
y_pred = pd.DataFrame(data=y_pred,columns=sample.columns)
 env.predict(y_pred)
 return y_pred 
<load_pretrained>
submission = predicting(X_test, df_model)
 submission
<prepare_x_and_y><EOS> 
submission.to_csv("submission.csv", index=False )