Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<feature_engineering>
import numpy as np
import pandas as pd
import os
data['date'] = pd.to_datetime(data['date'])
 data['year'] = data['date'].dt.year
data['month'] = data['date'].dt.month
data['week'] = data['date'].dt.week
data['day'] = data['date'].dt.day
data['dayofweek'] = data['date'].dt.dayofweek<feature_engineering>
import json
import tensorflow as tf
from matplotlib import pyplot as plt
def weekend(arg):
if arg==5 or arg==6:
return 1
else:
return 0
data['isweekend'] = data['dayofweek'].apply(weekend )<feature_engineering>
train_data = pd.read_json('/kaggle/input/stanford-covid-vaccine/train.json', lines = True)
 test_data = pd.read_json('/kaggle/input/stanford-covid-vaccine/test.json', lines = True)
 submission_format = pd.read_csv('/kaggle/input/stanford-covid-vaccine/sample_submission.csv', encoding = 'utf-8-sig' )
data['revenue'] = data['demand'] * data['sell_price']
data['lag_revenue_t1'] = data.groupby(['id'])['revenue'].transform(lambda x: x.shift(28))
data['rolling_revenue_std_t28'] = data.groupby(['id'])['lag_revenue_t1'].transform(lambda x: x.rolling(28 ).std())
 data['rolling_revenue_mean_t28'] = data.groupby(['id'])['lag_revenue_t1'].transform(lambda x: x.rolling(28 ).mean())
 data.drop(['revenue'],axis=1,inplace=True )<define_variables>
train_data.groupby(['SN_filter'] ).size()
features = ['item_id', 'cat_id', 'state_id', 'year', 'month', 'week', 'day', 'dayofweek', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2',
'snap_CA', 'snap_TX', 'snap_WI', 'sell_price', 'lag_t28', 'lag_t29', 'lag_t30', 'rolling_mean_t7', 'rolling_std_t7', 'rolling_mean_t30', 'rolling_mean_t90',
'rolling_mean_t180', 'rolling_std_t30', 'price_change_t1', 'price_change_t365', 'rolling_price_std_t7', 'rolling_price_std_t30', 'rolling_skew_t30', 'rolling_kurt_t30',
'isweekend','lag_revenue_t1','rolling_revenue_std_t28','rolling_revenue_mean_t28'
]
data = reduce_mem_usage(data )<prepare_x_and_y>
print('Training data:
',train_data['seq_scored'].value_counts())
 print('Test data:
',test_data['seq_scored'].value_counts())
 len(train_data['reactivity'].iloc[0] )
x_train = data[data['part'] == 'train']
y_train = x_train['demand']
x_val = data[(data['date'] > '2016-04-24')&(data['part'] == 'train')]
y_val = x_val['demand']
test = data[data['date'] > '2016-04-24']
test.loc[test['part']=='train','id'] = test.loc[test['part']=='train','id'].str.replace('_evaluation','_validation')
 x_train.drop(['demand','part',],inplace=True,axis=1)
 x_val.drop(['demand','part',],inplace=True,axis=1)
 test.drop(['demand','part',],inplace=True,axis=1 )<set_options>
flag = False
for i in range(0,len(train_data)) :
if(( [x<0 for x in train_data['reactivity_error'].iloc[i]].count(True)> 0)| 
([x<0 for x in train_data['deg_error_Mg_pH10'].iloc[i]].count(True)> 0)| 
([x<0 for x in train_data['deg_error_pH10'].iloc[i]].count(True)> 0)| 
([x<0 for x in train_data['deg_error_Mg_50C'].iloc[i]].count(True)> 0)| 
([x<0 for x in train_data['deg_error_50C'].iloc[i]].count(True)> 0)) :
flag = True
print(flag )
del data
gc.collect()<init_hyperparams>
min_reactivity_value = min(train_data['reactivity'].iloc[0])
 min_deg_Mg_pH10_value = min(train_data['deg_Mg_pH10'].iloc[0])
 min_deg_pH10_value = min(train_data['deg_pH10'].iloc[0])
 min_deg_Mg_50C_value = min(train_data['deg_Mg_50C'].iloc[0])
 min_deg_Mg_50C_value = min(train_data['deg_50C'].iloc[0])
 for i in range(0,len(train_data)) :
if(min(train_data['reactivity'].iloc[i])< min_reactivity_value):
min_reactivity_value = min(train_data['reactivity'].iloc[i])
 if(min(train_data['deg_Mg_pH10'].iloc[i])< min_deg_Mg_pH10_value):
min_deg_Mg_pH10_value = min(train_data['deg_Mg_pH10'].iloc[i])
 if(min(train_data['deg_pH10'].iloc[i])< min_deg_pH10_value):
min_deg_pH10_value = min(train_data['deg_pH10'].iloc[i])
 if(min(train_data['deg_Mg_50C'].iloc[i])< min_deg_Mg_50C_value):
min_deg_Mg_50C_value = min(train_data['deg_Mg_50C'].iloc[i])
 if(min(train_data['deg_50C'].iloc[i])< min_deg_Mg_50C_value):
min_deg_Mg_50C_value = min(train_data['deg_50C'].iloc[i])
 print(min_reactivity_value, min_deg_Mg_pH10_value, min_deg_pH10_value, min_deg_Mg_50C_value, min_deg_Mg_50C_value )
params = {
'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'n_jobs': -1,
'seed': 236,
'learning_rate': 0.1}<train_model>
TEST = None
for i in category:
for j in category2:
x_t,x_v,y_t,y_v,t = dataset(i,j)
 train_set = Dataset(x_t[features], y_t)
 val_set = Dataset(x_v[features], y_v)
 del x_t, y_t
gc.collect()
model = train(params, train_set, num_boost_round = 150, early_stopping_rounds = 40, valid_sets = [train_set, val_set], verbose_eval = 1000)
 y_pred = model.predict(t[features])
 t['demand'] = y_pred
TEST = pd.concat([TEST,t],axis=0)
 del x_v,y_v,t,train_set,val_set,y_pred,model
gc.collect()<prepare_x_and_y>
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
predictions1 = TEST[TEST['id'].apply(lambda x: "validation" in x)][['id', 'date', 'demand']]
predictions2 = TEST[TEST['id'].apply(lambda x: "evaluation" in x)][['id', 'date', 'demand']]<save_to_csv>
def read_bpps_sum(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).sum(axis=1))
return bpps_arr
def read_bpps_max(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).max(axis=1))
return bpps_arr
def read_bpps_nb(df):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps_arr = []
for mol_id in df.id.to_list() :
bpps = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy")
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
bpps_arr.append(bpps_nb)
 return bpps_arr
os.chdir("/kaggle/working/")
 train_data['bpps_sum'] = read_bpps_sum(train_data)
 test_data['bpps_sum'] = read_bpps_sum(test_data)
 train_data['bpps_max'] = read_bpps_max(train_data)
 test_data['bpps_max'] = read_bpps_max(test_data)
 train_data['bpps_nb'] = read_bpps_nb(train_data)
 test_data['bpps_nb'] = read_bpps_nb(test_data)
 train_data.head()
predictions = pd.concat([prediction1,prediction2],axis=0)
 predictions.to_csv("submissionstoreanddeptwise.csv",index=False )<load_from_csv>
def get_bases(data):
bases = []
for j in range(len(data)) :
counts = dict(count(data.iloc[j]['sequence']))
bases.append((
counts['A'] / 107,
counts['G'] / 107,
counts['C'] / 107,
counts['U'] / 107 
))
bases = pd.DataFrame(bases, columns=['A_percent', 'G_percent', 'C_percent', 'U_percent'])
 return bases
!gzip submissionstoreanddeptwise.csv<set_options>
def get_pairs_rate(data):
pairs_rate = []
for j in range(len(data)) :
res = dict(count(data.iloc[j]['structure']))
pairs_rate.append(res['('] / 53.5)
 pairs_rate = pd.DataFrame(pairs_rate, columns=['pairs_rate'])
 return pairs_rate
warnings.filterwarnings('ignore' )<randomize_order>
def get_pairs(data):
pairs = []
all_partners = []
for j in range(len(data)) :
partners = [-1 for i in range(130)]
pairs_dict = {}
queue = []
for i in range(0, len(data.iloc[j]['structure'])) :
if data.iloc[j]['structure'][i] == '(':
queue.append(i)
 if data.iloc[j]['structure'][i] == ')':
first = queue.pop()
try:
pairs_dict[(data.iloc[j]['sequence'][first], data.iloc[j]['sequence'][i])] += 1
except:
pairs_dict[(data.iloc[j]['sequence'][first], data.iloc[j]['sequence'][i])] = 1
partners[first] = i
partners[i] = first
all_partners.append(partners)
 pairs_num = 0
pairs_unique = [('U', 'G'),('C', 'G'),('U', 'A'),('G', 'C'),('A', 'U'),('G', 'U')]
for item in pairs_dict:
pairs_num += pairs_dict[item]
add_tuple = list()
for item in pairs_unique:
try:
add_tuple.append(pairs_dict[item]/pairs_num)
 except:
add_tuple.append(0)
 pairs.append(add_tuple)
 pairs = pd.DataFrame(pairs, columns=['U-G', 'C-G', 'U-A', 'G-C', 'A-U', 'G-U'])
 return pairs
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 def df_parallelize_run(func, t_split):
num_cores = np.min([N_CORES,len(t_split)])
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, t_split), axis=1)
 pool.close()
pool.join()
return df<categorify>
def get_loops(data):
loops = []
for j in range(len(data)) :
counts = dict(count(data.iloc[j]['predicted_loop_type']))
available = ['E', 'S', 'H', 'B', 'X', 'I', 'M']
row = []
for item in available:
try:
row.append(counts[item] / 107)
 except:
row.append(0)
 loops.append(row)
 loops = pd.DataFrame(loops, columns=available)
 return loops
def get_data_by_store(store):
df = pd.concat([pd.read_pickle(BASE),
pd.read_pickle(PRICE ).iloc[:,2:],
pd.read_pickle(CALENDAR ).iloc[:,2:]],
axis=1)
 df = df[df['store_id']==store]
df2 = pd.read_pickle(MEAN_ENC)[mean_features]
df2 = df2[df2.index.isin(df.index)]
df3 = pd.read_pickle(LAGS ).iloc[:,3:]
df3 = df3[df3.index.isin(df.index)]
df = pd.concat([df, df2], axis=1)
 del df2
df = pd.concat([df, df3], axis=1)
 del df3
features = [col for col in list(df)if col not in remove_features]
df = df[['id','d',TARGET]+features]
df = df[df['d']>=START_TRAIN].reset_index(drop=True)
 return df, features
def get_base_test() :
base_test = pd.DataFrame()
for store_id in STORES_IDS:
temp_df = pd.read_pickle('.. /input/darkerevapretrained/test_'+store_id+'.pkl')
 temp_df['store_id'] = store_id
base_test = pd.concat([base_test, temp_df] ).reset_index(drop=True)
 return base_test
def make_lag(LAG_DAY):
lag_df = base_test[['id','d',TARGET]]
col_name = 'sales_lag_'+str(LAG_DAY)
 lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(LAG_DAY)).astype(np.float16)
 return lag_df[[col_name]]
def make_lag_roll(LAG_DAY):
shift_day = LAG_DAY[0]
roll_wind = LAG_DAY[1]
lag_df = base_test[['id','d',TARGET]]
col_name = 'rolling_mean_tmp_'+str(shift_day)+'_'+str(roll_wind)
 lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(shift_day ).rolling(roll_wind ).mean())
 return lag_df[[col_name]]<init_hyperparams>
def get_structure_adj(train):
Ss = []
for i in tqdm(range(len(train))):
seq_length = train["seq_length"].iloc[i]
structure = train["structure"].iloc[i]
sequence = train["sequence"].iloc[i]
cue = []
a_structures = { 
("A", "U"): np.zeros([seq_length, seq_length]), 
("C", "G"): np.zeros([seq_length, seq_length]), 
("U", "G"): np.zeros([seq_length, seq_length]), 
("U", "A"): np.zeros([seq_length, seq_length]), 
("G", "C"): np.zeros([seq_length, seq_length]), 
("G", "U"): np.zeros([seq_length, seq_length]),
}
a_structure = np.zeros([seq_length, seq_length])
 for j in range(seq_length):
if structure[j] == "(":
cue.append(j)
 elif structure[j] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[j])][start, j] = 1
a_structures[(sequence[j], sequence[start])][j, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis = 2)
 a_strc = np.sum(a_strc, axis = 2, keepdims = True)
 Ss.append(a_strc)
 Ss = np.array(Ss)
 print(Ss.shape)
 return Ss
lgb_params = {
'boosting_type': 'gbdt',
'metric': 'custom',
'objective': 'rmse',
'n_jobs': -1,
'n_estimators': 3000,
'verbose': -1,
}<init_hyperparams>
As = []
data = train_data[train_data['signal_to_noise'] > 1].copy()
for id in tqdm(data['id']):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As.append(a)
 As = np.array(As )
VER = 1
SEED = 42
seed_everything(SEED)
 lgb_params['seed'] = SEED
N_CORES = psutil.cpu_count()
TARGET = 'sales'
START_TRAIN = 0
END_TRAIN = 1941
P_HORIZON = 28
USE_AUX = True
remove_features = ['id','state_id','store_id',
'date','wm_yr_wk','d',TARGET]
mean_features = ['enc_cat_id_mean','enc_cat_id_std',
'enc_dept_id_mean','enc_dept_id_std',
'enc_item_id_mean','enc_item_id_std']
ORIGINAL = '.. /input/m5-forecasting-accuracy/'
BASE = '.. /input/evalm5/grid_part_1.pkl'
PRICE = '.. /input/evalm5/grid_part_2.pkl'
CALENDAR = '.. /input/evalm5/grid_part_3.pkl'
LAGS = '.. /input/evalm5/lags_df_28.pkl'
MEAN_ENC = '.. /input/evalm5/mean_encoding_df.pkl'
AUX_MODELS = '.. /input/darkerevapretrained/'
STORES_IDS = pd.read_csv('.. /input/m5-forecasting-accuracy/sales_train_validation.csv')['store_id']
STORES_IDS = list(STORES_IDS.unique())
 SHIFT_DAY = 28
N_LAGS = 15
LAGS_SPLIT = [col for col in range(SHIFT_DAY,SHIFT_DAY+N_LAGS)]
ROLS_SPLIT = []
for i in [1,7,14]:
for j in [7,14,30,60]:
ROLS_SPLIT.append([i,j] )<define_variables>
def get_distance_matrix(As):
idx = np.arange(As.shape[1])
 Ds = []
for i in range(len(idx)) :
d = np.abs(idx[i] - idx)
 Ds.append(d)
 Ds = np.array(Ds)+ 1
Ds = 1/Ds
Ds = Ds[None, :,:]
Ds = np.repeat(Ds, len(As), axis = 0)
 Dss = []
for i in [1, 2, 4]:
Dss.append(Ds ** i)
 Ds = np.stack(Dss, axis = 3)
 print(Ds.shape)
 return Ds
MODEL_FEATURES = ['item_id',
'dept_id',
'cat_id',
'release',
'sell_price',
'price_max',
'price_min',
'price_std',
'price_mean',
'price_norm',
'price_nunique',
'item_nunique',
'price_momentum',
'price_momentum_m',
'price_momentum_y',
'event_name_1',
'event_type_1',
'event_name_2',
'event_type_2',
'snap_CA',
'snap_TX',
'snap_WI',
'tm_d',
'tm_w',
'tm_m',
'tm_y',
'tm_wm',
'tm_dw',
'tm_w_end',
'enc_cat_id_mean',
'enc_cat_id_std',
'enc_dept_id_mean',
'enc_dept_id_std',
'enc_item_id_mean',
'enc_item_id_std',
'sales_lag_1',
'sales_lag_2',
'sales_lag_3',
'sales_lag_4',
'sales_lag_5',
'sales_lag_6',
'sales_lag_7',
'sales_lag_8',
'sales_lag_9',
'sales_lag_10',
'sales_lag_11',
'sales_lag_12',
'sales_lag_13',
'sales_lag_14',
'sales_lag_15',
'sales_lag_16',
'sales_lag_17',
'sales_lag_18',
'sales_lag_19',
'sales_lag_20',
'sales_lag_21',
'sales_lag_22',
'sales_lag_23',
'sales_lag_24',
'sales_lag_25',
'sales_lag_26',
'sales_lag_27',
'rolling_mean_7',
'rolling_std_7',
'rolling_mean_14',
'rolling_std_14',
'rolling_mean_30',
'rolling_std_30',
'rolling_mean_60',
'rolling_std_60',
'rolling_mean_180',
'rolling_std_180',
'rolling_mean_tmp_1_7',
'rolling_mean_tmp_1_14',
'rolling_mean_tmp_1_30',
'rolling_mean_tmp_1_60',
'rolling_mean_tmp_7_7',
'rolling_mean_tmp_7_14',
'rolling_mean_tmp_7_30',
'rolling_mean_tmp_7_60',
'rolling_mean_tmp_14_7',
'rolling_mean_tmp_14_14',
'rolling_mean_tmp_14_30',
'rolling_mean_tmp_14_60',
'cluster']<define_variables>
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type'], seq_length = 107, flag = 'train'):
base_fea = np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 bpps_sum_fea = np.array(df['bpps_sum'].to_list())[:,:,np.newaxis]
bpps_max_fea = np.array(df['bpps_max'].to_list())[:,:,np.newaxis]
bpps_nb_fea = np.array(df['bpps_nb'].to_list())[:,:,np.newaxis]
Ss = get_structure_adj(df)
 Ss = Ss.sum(axis = 1)
 if flag == 'train':
Ds = get_distance_matrix(As)
 elif flag == 'test_private':
Ds = get_distance_matrix(As_private)
 elif flag == 'test_public':
Ds = get_distance_matrix(As_public)
 Ds = Ds.sum(axis = 1)
 data = np.concatenate([base_fea,bpps_sum_fea,bpps_max_fea,bpps_nb_fea, Ss, Ds], 2)
 array_data =(np.reshape(( [([list(df['A_percent'])[0]] * seq_length)]),(1,seq_length,1)))
 for i in range(1,len(df)) :
array_data_i =(np.reshape(( [([list(df['A_percent'])[i]] * seq_length)]),(1,seq_length,1)))
 array_data = np.concatenate([array_data,array_data_i], axis = 0)
 data = np.concatenate([data, array_data], 2)
 for col in ['G_percent','C_percent','U_percent','U-G','C-G','U-A','G-C','A-U','G-U',
'E','S','H','B','X','I','M','pairs_rate']:
array_data =(np.reshape(( [([list(df[col])[0]] * seq_length)]),(1,seq_length,1)))
 for i in range(1,len(df)) :
arraydom_data_i =(np.reshape(( [([list(df[col])[i]] * seq_length)]),(1,seq_length,1)))
 array_data = np.concatenate([array_data,array_data_i], axis = 0)
 data = np.concatenate([data, array_data], 2)
 return data
all_preds = pd.DataFrame()
base_test = get_base_test()
main_time = time.time()
for PREDICT_DAY in range(1,29):
print('Predict | Day:', PREDICT_DAY)
 start_time = time.time()
grid_df = base_test.copy()
grid_df = pd.concat([grid_df, df_parallelize_run(make_lag_roll, ROLS_SPLIT)], axis=1)
 print('Predict | Day:', PREDICT_DAY)
 for store_id in tqdm(STORES_IDS):
model_path = 'lgb_model_'+store_id+'_v'+str(VER)+'.bin'
if USE_AUX:
model_path = AUX_MODELS + model_path
estimator = pickle.load(open(model_path, 'rb')) [0]
day_mask = base_test['d']==(END_TRAIN+PREDICT_DAY)
 store_mask = base_test['store_id']==store_id
mask =(day_mask)&(store_mask)
 base_test[TARGET][mask] = estimator.predict(grid_df[mask][MODEL_FEATURES], num_iteration = estimator.best_iteration)
 temp_df = base_test[day_mask][['id',TARGET]]
temp_df.columns = ['id','F'+str(PREDICT_DAY)]
if 'id' in list(all_preds):
all_preds = all_preds.merge(temp_df, on=['id'], how='left')
 else:
all_preds = temp_df.copy()
print('
' %0.2f min total |' %(( time.time() - main_time)/ 60),
' %0.2f day sales |' %(temp_df['F'+str(PREDICT_DAY)].sum()))
del temp_df
all_preds = all_preds.reset_index(drop=True )<save_to_csv>
bases = get_bases(train_data)
 pairs = get_pairs(train_data)
 loops = get_loops(train_data)
 pairs_rate = get_pairs_rate(train_data)
 train_data = pd.concat([train_data, bases, pairs, loops, pairs_rate], axis=1)
 bases = get_bases(test_data)
 pairs = get_pairs(test_data)
 loops = get_loops(test_data)
 pairs_rate = get_pairs_rate(test_data)
 test_data = pd.concat([test_data, bases, pairs, loops, pairs_rate], axis=1 )
submission = pd.read_csv(ORIGINAL+'sample_submission.csv')[['id']]
submission = submission.merge(all_preds, on=['id'], how='left' ).fillna(0)
 submission.to_csv('darker_eval_v'+str(VER)+'.csv', index=False )<import_modules>
train_inputs = preprocess_inputs(train_data.loc[train_data['signal_to_noise'] > 1], seq_length = 107, flag = 'train')
 train_labels = np.array(train_data.loc[train_data['signal_to_noise'] > 1][target_cols].values.tolist() ).transpose(( 0, 2, 1))
from datetime import datetime, timedelta
import gc
import numpy as np, pandas as pd
import lightgbm as lgb<define_variables>
def root_mean_squared_error(y_true, y_pred):
return tf.sqrt(mean_squared_error(y_true, y_pred))
def MCRMSE(y_true, y_pred):
colwise_mse = tf.reduce_mean(tf.square(y_true - y_pred), axis=1)
 return tf.reduce_mean(tf.sqrt(colwise_mse), axis=1)
 def lstm_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.LSTM(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer = 'orthogonal'))
def gru_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.GRU(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer='orthogonal'))
def build_model(n_layers = 2, seq_len = 107, num_features = 28, embed_dim = 200, sp_dropout = 0.2, hidden_dim = 512, dropout = 0.5, pred_len = 68, gru_flag = False):
inputs = tf.keras.layers.Input(shape=(seq_len, num_features))
categorical_feats = inputs[:, :, :3]
numerical_feats = inputs[:, :, 3:10]
overall_gene_feats = inputs[:, :, 10:]
embed = tf.keras.layers.Embedding(input_dim=len(token2int), output_dim=embed_dim )(categorical_feats)
 reshaped = tf.reshape(embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped_1 = tf.keras.layers.concatenate([reshaped, numerical_feats], axis=2)
 normalized_layer_1 = tf.keras.layers.BatchNormalization()(reshaped_1)
 spatial_dropout = tf.keras.layers.SpatialDropout1D(sp_dropout )(normalized_layer_1)
 if gru_flag:
for x in range(n_layers):
normalized_layer_1 = gru_layer(hidden_dim, dropout )(normalized_layer_1)
 normalized_layer_2 = tf.keras.layers.BatchNormalization()(normalized_layer_1)
 else:
for x in range(n_layers):
normalized_layer_1 = lstm_layer(hidden_dim, dropout )(normalized_layer_1)
 normalized_layer_2 = tf.keras.layers.BatchNormalization()(normalized_layer_1)
 concat_layer = tf.keras.layers.concatenate([normalized_layer_2, overall_gene_feats], axis=2)
 dense_layer_1 = tf.keras.layers.Dense(100, activation = 'linear' )(concat_layer)
 normalized_layer_3 = tf.keras.layers.BatchNormalization()(dense_layer_1)
 dropout_layer_1 = tf.keras.layers.SpatialDropout1D(sp_dropout )(normalized_layer_3)
 dense_layer_2 = tf.keras.layers.Dense(100, activation = 'linear' )(dropout_layer_1)
 normalized_layer_4 = tf.keras.layers.BatchNormalization()(dense_layer_2)
 dropout_layer_2 = tf.keras.layers.SpatialDropout1D(sp_dropout )(normalized_layer_4)
 truncated = dropout_layer_2[:, :pred_len]
out = tf.keras.layers.Dense(5, activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 adam = tf.optimizers.Adam()
model.compile(optimizer = adam, loss = MCRMSE)
 return model
CAL_DTYPES={"event_name_1": "category", "event_name_2": "category", "event_type_1": "category",
"event_type_2": "category", "weekday": "category", 'wm_yr_wk': 'int16', "wday": "int16",
"month": "int16", "year": "int16", "snap_CA": "float32", 'snap_TX': 'float32', 'snap_WI': 'float32' }
PRICE_DTYPES = {"store_id": "category", "item_id": "category", "wm_yr_wk": "int16","sell_price":"float32" }<set_options>
public_df = test_data.query("seq_length == 107" ).copy()
private_df = test_data.query("seq_length == 130" ).copy()
pd.options.display.max_columns = 50<define_variables>
As_public = []
for id in tqdm(public_df["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_public.append(a)
 As_public = np.array(As_public)
 As_private = []
for id in tqdm(private_df["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_private.append(a)
 As_private = np.array(As_private )
h = 28
max_lags = 57
tr_last = 1913 + 28
fday = datetime(2016,4, 25)+ timedelta(days= 28)
 fday,tr_last,max_lags<load_from_csv>
model_LSTM_on_test_data_public = build_model(seq_len=107, pred_len=107, gru_flag = False)
 model_LSTM_on_test_data_public.load_weights('.. /input/openvaccine-covid-model-weights/LSTM model.h5')
 pred_test_data_public_LSTM = model_LSTM_on_test_data_public.predict(public_inputs)
 model_GRU_on_test_data_public = build_model(seq_len=107, pred_len=107, gru_flag = True)
 model_GRU_on_test_data_public.load_weights('.. /input/openvaccine-covid-model-weights/GRU model(1 ).h5')
 pred_test_data_public_GRU = model_GRU_on_test_data_public.predict(public_inputs )
def create_dt(is_train = True, nrows = None, first_day = 1200):
prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv", dtype = PRICE_DTYPES)
 for col, col_dtype in PRICE_DTYPES.items() :
if col_dtype == "category":
prices[col] = prices[col].cat.codes.astype("int16")
 prices[col] -= prices[col].min()
cal = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv", dtype = CAL_DTYPES)
 cal["date"] = pd.to_datetime(cal["date"])
 for col, col_dtype in CAL_DTYPES.items() :
if col_dtype == "category":
cal[col] = cal[col].cat.codes.astype("int16")
 cal[col] -= cal[col].min()
start_day = max(1 if is_train else tr_last-max_lags, first_day)
 numcols = [f"d_{day}" for day in range(start_day,tr_last+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 dt = pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_evaluation.csv",
nrows = nrows, usecols = catcols + numcols, dtype = dtype)
 for col in catcols:
if col != "id":
dt[col] = dt[col].cat.codes.astype("int16")
 dt[col] -= dt[col].min()
if not is_train:
for day in range(tr_last+1, tr_last+ 28 +1):
dt[f"d_{day}"] = np.nan
dt = pd.melt(dt,
id_vars = catcols,
value_vars = [col for col in dt.columns if col.startswith("d_")],
var_name = "d",
value_name = "sales")
 dt = dt.merge(cal, on= "d", copy = False)
 dt = dt.merge(prices, on = ["store_id", "item_id", "wm_yr_wk"], copy = False)
 return dt<categorify>
model_LSTM_on_test_data_private = build_model(seq_len=130, pred_len=130, gru_flag = False)
 model_LSTM_on_test_data_private.load_weights('.. /input/openvaccine-covid-model-weights/LSTM model.h5')
 pred_test_data_private_LSTM = model_LSTM_on_test_data_private.predict(private_inputs)
 model_GRU_on_test_data_private = build_model(seq_len=130, pred_len=130, gru_flag = True)
 model_GRU_on_test_data_private.load_weights('.. /input/openvaccine-covid-model-weights/GRU model(1 ).h5')
 pred_test_data_private_GRU = model_GRU_on_test_data_private.predict(private_inputs )
def create_fea(dt):
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
dt[lag_col] = dt[["id","sales"]].groupby("id")["sales"].shift(lag)
 wins = [7, 28]
for win in wins :
for lag,lag_col in zip(lags, lag_cols):
dt[f"rmean_{lag}_{win}"] = dt[["id", lag_col]].groupby("id")[lag_col].transform(lambda x : x.rolling(win ).mean())
 date_features = {
"wday": "weekday",
"week": "weekofyear",
"month": "month",
"quarter": "quarter",
"year": "year",
"mday": "day",
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt["date"].dt, date_feat_func ).astype("int16" )<define_variables>
def format_predictions(public_preds, private_preds):
preds = []
for df, preds_ in [(public_df, public_preds),(private_df, private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds_[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds.append(single_df)
 return pd.concat(preds ).reset_index(drop = True )
FIRST_DAY = 350<correct_missing_values>
lstm_preds = format_predictions(pred_test_data_public_LSTM, pred_test_data_private_LSTM)
 gru_preds = format_predictions(pred_test_data_public_GRU, pred_test_data_private_GRU )
df.dropna(inplace = True)
 df.shape<prepare_x_and_y>
submission_LSTM = submission_format[['id_seqpos']].merge(lstm_preds, how = 'inner', on = 'id_seqpos')
 submission_GRU = submission_format[['id_seqpos']].merge(gru_preds, how = 'inner', on = 'id_seqpos' )
cat_feats = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id'] + ["event_name_1", "event_name_2", "event_type_1", "event_type_2"]
useless_cols = ["id", "date", "sales","d", "wm_yr_wk", "weekday"]
train_cols = df.columns[~df.columns.isin(useless_cols)]
X_train = df[train_cols]
y_train = df["sales"]<create_dataframe>
submission_lstm_gru_combined = submission_GRU.merge(submission_LSTM, how = 'inner', on = 'id_seqpos')
 gru_weight = 0.5
lstm_weight = 0.5
for i in range(len(target_cols)) :
submission_lstm_gru_combined[target_cols[i]] = submission_lstm_gru_combined[target_cols[i]+'_x']*gru_weight + submission_lstm_gru_combined[target_cols[i]+'_y']*lstm_weight
<create_dataframe>
submission_lstm_gru_combined = submission_lstm_gru_combined[['id_seqpos'] + target_cols]
<prepare_x_and_y><EOS> 
os.chdir("/kaggle/working/")
 submission_LSTM.to_csv('submission_LSTM.csv', index = False)
 submission_GRU.to_csv('submission_GRU.csv', index = False)
 submission_lstm_gru_combined.to_csv('submission_lstm_gru_combined.csv', index = False )