Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<define_variables>
%matplotlib inline
warnings.filterwarnings('ignore' )
h = 28
max_lags = 57
tr_last = 1913 + 28
fday = datetime(2016,4, 25)+ timedelta(days= 28)
 fday,tr_last,max_lags<load_from_csv>
train = pd.read_json(".. /input/stanford-covid-vaccine/train.json",lines=True)
 test = pd.read_json(".. /input/stanford-covid-vaccine/test.json",lines=True)
 sub = pd.read_csv(".. /input/stanford-covid-vaccine/sample_submission.csv")
 test_pub = test[test["seq_length"] == 107]
test_pri = test[test["seq_length"] == 130]
def create_dt(is_train = True, nrows = None, first_day = 1200):
prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv", dtype = PRICE_DTYPES)
 for col, col_dtype in PRICE_DTYPES.items() :
if col_dtype == "category":
prices[col] = prices[col].cat.codes.astype("int16")
 prices[col] -= prices[col].min()
cal = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv", dtype = CAL_DTYPES)
 cal["date"] = pd.to_datetime(cal["date"])
 for col, col_dtype in CAL_DTYPES.items() :
if col_dtype == "category":
cal[col] = cal[col].cat.codes.astype("int16")
 cal[col] -= cal[col].min()
start_day = max(1 if is_train else tr_last-max_lags, first_day)
 numcols = [f"d_{day}" for day in range(start_day,tr_last+1)]
catcols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numcols}
dtype.update({col: "category" for col in catcols if col != "id"})
 dt = pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_evaluation.csv",
nrows = nrows, usecols = catcols + numcols, dtype = dtype)
 for col in catcols:
if col != "id":
dt[col] = dt[col].cat.codes.astype("int16")
 dt[col] -= dt[col].min()
if not is_train:
for day in range(tr_last+1, tr_last+ 28 +1):
dt[f"d_{day}"] = np.nan
dt = pd.melt(dt,
id_vars = catcols,
value_vars = [col for col in dt.columns if col.startswith("d_")],
var_name = "d",
value_name = "sales")
 dt = dt.merge(cal, on= "d", copy = False)
 dt = dt.merge(prices, on = ["store_id", "item_id", "wm_yr_wk"], copy = False)
 return dt<categorify>
aug_df = pd.read_csv('.. /input/covid19-mrna-augmentation-data-and-features/aug_data1.csv')
 aug_df = aug_df.drop_duplicates(subset=['id', 'structure'])
 def aug_data(df):
target_df = df.copy()
new_df = aug_df[aug_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id','sequence'], how='left')
 df['cnt'] = df['id'].map(new_df[['id','cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
df = df.append(new_df[df.columns])
 return df
train = aug_data(train ).reset_index(drop=True)
 test_pub = aug_data(test_pub ).reset_index(drop=True)
 test_pri = aug_data(test_pri ).reset_index(drop=True )
def create_fea(dt):
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
dt[lag_col] = dt[["id","sales"]].groupby("id")["sales"].shift(lag)
 wins = [7, 28]
for win in wins :
for lag,lag_col in zip(lags, lag_cols):
dt[f"rmean_{lag}_{win}"] = dt[["id", lag_col]].groupby("id")[lag_col].transform(lambda x : x.rolling(win ).mean())
 date_features = {
"wday": "weekday",
"week": "weekofyear",
"month": "month",
"quarter": "quarter",
"year": "year",
"mday": "day",
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt["date"].dt, date_feat_func ).astype("int16" )<define_variables>
As = []
for id in tqdm(train["id"]):
a = np.load(f".. /input/stanford-covid-vaccine/bpps/{id}.npy" ).astype(np.float16)
 As.append(a)
 As = np.array(As)
 As_pub = []
for id in tqdm(test_pub["id"]):
a = np.load(f".. /input/stanford-covid-vaccine/bpps/{id}.npy" ).astype(np.float16)
 As_pub.append(a)
 As_pub = np.array(As_pub)
 As_pri = []
for id in tqdm(test_pri["id"]):
a = np.load(f".. /input/stanford-covid-vaccine/bpps/{id}.npy" ).astype(np.float16)
 As_pri.append(a)
 As_pri = np.array(As_pri)
 As_cf = []
for id in tqdm(train["id"]):
a = np.load(f".. /input/covid19-mrna-augmentation-data-and-features/bpps_contrafold_gamma1/bpps_contrafold_gamma1/bpps_contrafold_gamma1/{id}.npy" ).astype(np.float16)
 As_cf.append(a)
 As_cf = np.array(As_cf)
 As_pub_cf = []
for id in tqdm(test_pub["id"]):
a = np.load(f".. /input/covid19-mrna-augmentation-data-and-features/bpps_contrafold_gamma1/bpps_contrafold_gamma1/bpps_contrafold_gamma1/{id}.npy" ).astype(np.float16)
 As_pub_cf.append(a)
 As_pub_cf = np.array(As_pub_cf)
 As_pri_cf = []
for id in tqdm(test_pri["id"]):
a = np.load(f".. /input/covid19-mrna-augmentation-data-and-features/bpps_contrafold_gamma1/bpps_contrafold_gamma1/bpps_contrafold_gamma1/{id}.npy" ).astype(np.float16)
 As_pri_cf.append(a)
 As_pri_cf = np.array(As_pri_cf)
 As_rs = []
for id in tqdm(train["id"]):
a = np.load(f".. /input/covid19-mrna-augmentation-data-and-features/bpps_rnasoft_gamma1/bpps_rnasoft_gamma1/bpps_rnasoft_gamma1/{id}.npy" ).astype(np.float16)
 As_rs.append(a)
 As_rs = np.array(As_rs)
 As_pub_rs = []
for id in tqdm(test_pub["id"]):
a = np.load(f".. /input/covid19-mrna-augmentation-data-and-features/bpps_rnasoft_gamma1/bpps_rnasoft_gamma1/bpps_rnasoft_gamma1/{id}.npy" ).astype(np.float16)
 As_pub_rs.append(a)
 As_pub_rs = np.array(As_pub_rs)
 As_pri_rs = []
for id in tqdm(test_pri["id"]):
a = np.load(f".. /input/covid19-mrna-augmentation-data-and-features/bpps_rnasoft_gamma1/bpps_rnasoft_gamma1/bpps_rnasoft_gamma1/{id}.npy" ).astype(np.float16)
 As_pri_rs.append(a)
 As_pri_rs = np.array(As_pri_rs )
FIRST_DAY = 350<correct_missing_values>
def get_structure_adj(train):
Ss = []
for i in tqdm(range(len(train))):
seq_length = train["seq_length"].iloc[i]
structure = train["structure"].iloc[i]
sequence = train["sequence"].iloc[i]
cue = []
a_structures = { 
("A", "U"): np.zeros([seq_length, seq_length]), 
("C", "G"): np.zeros([seq_length, seq_length]), 
("U", "G"): np.zeros([seq_length, seq_length]), 
("U", "A"): np.zeros([seq_length, seq_length]), 
("G", "C"): np.zeros([seq_length, seq_length]), 
("G", "U"): np.zeros([seq_length, seq_length]),
}
a_structure = np.zeros([seq_length, seq_length])
 for i in range(seq_length):
if structure[i] == "(":
cue.append(i)
 elif structure[i] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[i])][start, i] = 1
a_structures[(sequence[i], sequence[start])][i, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis = 2)
 a_strc = np.sum(a_strc, axis = 2, keepdims = True)
 Ss.append(a_strc)
 Ss = np.array(Ss)
 return Ss.astype(np.float16)
 Ss = get_structure_adj(train)
 Ss_pub = get_structure_adj(test_pub)
 Ss_pri = get_structure_adj(test_pri )
df.dropna(inplace = True)
 df.shape<prepare_x_and_y>
def get_distance_matrix(As):
idx = np.arange(As.shape[1])
 Ds = []
for i in range(len(idx)) :
d = np.abs(idx[i] - idx)
 Ds.append(d)
 Ds = np.array(Ds)+ 1
Ds = Ds / Ds.shape[1]
Ds = Ds[None, :,:]
Ds = np.repeat(Ds, len(As), axis = 0)
 Dss = []
for i in [1]:
Dss.append(Ds ** i)
 Ds = np.stack(Dss, axis = 3)
 return Ds.astype(np.float16)
 Ds = get_distance_matrix(As)
 Ds_pub = get_distance_matrix(As_pub)
 Ds_pri = get_distance_matrix(As_pri )
cat_feats = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id'] + ["event_name_1", "event_name_2", "event_type_1", "event_type_2"]
useless_cols = ["id", "date", "sales","d", "wm_yr_wk", "weekday"]
train_cols = df.columns[~df.columns.isin(useless_cols)]
X_train = df[train_cols]
y_train = df["sales"]<create_dataframe>
As = np.concatenate([As[:,:,:,None], As_cf[:,:,:,None], As_rs[:,:,:,None], Ss, Ds], axis = 3 ).astype(np.float16)
 del Ss, Ds, As_cf, As_rs
As_pub = np.concatenate([As_pub[:,:,:,None], As_pub_cf[:,:,:,None], As_pub_rs[:,:,:,None], Ss_pub, Ds_pub], axis = 3 ).astype(np.float16)
 del Ss_pub, Ds_pub, As_pub_cf, As_pub_rs
As_pri = np.concatenate([As_pri[:,:,:,None], As_pri_cf[:,:,:,None], As_pri_rs[:,:,:,None], Ss_pri, Ds_pri], axis = 3 ).astype(np.float16)
 del Ss_pri, Ds_pri, As_pri_cf, As_pri_rs
print(As.shape, As_pub.shape, As_pri.shape )
%%time
np.random.seed(777)
 fake_valid_inds = np.random.choice(X_train.index.values, 2_000_000, replace = False)
 train_inds = np.setdiff1d(X_train.index.values, fake_valid_inds)
 train_data = lgb.Dataset(X_train.loc[train_inds] , label = y_train.loc[train_inds],
categorical_feature=cat_feats, free_raw_data=False)
 fake_valid_data = lgb.Dataset(X_train.loc[fake_valid_inds], label = y_train.loc[fake_valid_inds],
categorical_feature=cat_feats,
free_raw_data=False )<prepare_x_and_y>
f, ax = plt.subplots(1, 5, figsize=(15, 3))
for i in range(As.shape[-1]):
ax[i].imshow(As[0, :, :, i].astype(np.float32))
plt.show()
del df, X_train, y_train, fake_valid_inds,train_inds ; gc.collect()<init_hyperparams>
arnie_train_features = pickle.load(open('.. /input/covid19-mrna-augmentation-data-and-features/arnie/arnie/train_features.p', 'rb'))
arnie_test_features = pickle.load(open('.. /input/covid19-mrna-augmentation-data-and-features/arnie/arnie/test_features.p', 'rb'))
capr_train_features = pickle.load(open('.. /input/covid19-mrna-augmentation-data-and-features/CapR_train.pickle', 'rb'))
capr_test_features = pickle.load(open('.. /input/covid19-mrna-augmentation-data-and-features/CapR_test.pickle', 'rb'))
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : 1500,
'num_leaves': 128,
"min_data_in_leaf": 100,
}<train_model>
def return_ohe(n, i):
tmp = [0] * n
tmp[i] = 1
return tmp
def get_input(train):
mapping = {}
vocab = ["A", "G", "C", "U"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_node = np.stack(train["sequence"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 mapping = {}
vocab = ["S", "M", "I", "B", "H", "E", "X"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_loop = np.stack(train["predicted_loop_type"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 X_pos1 = np.stack(train["sequence"].apply(lambda x : np.arange(len(x)) % 3 == 0)) [:,:,np.newaxis]
X_pos2 = np.stack(train["sequence"].apply(lambda x : np.arange(len(x)) % 3 == 1)) [:,:,np.newaxis]
X_pos3 = np.stack(train["sequence"].apply(lambda x : np.arange(len(x)) % 3 == 2)) [:,:,np.newaxis]
X_certainty = np.zeros(( X_pos1.shape[:2] +(3,)))
 X_loop_capr = np.zeros(( X_pos1.shape[:2] +(6,)))
 X_bpp_sum = np.zeros(( X_pos1.shape[:2] +(1,)))
 X_bpp_nb_zeroes = np.zeros(( X_pos1.shape[:2] +(1,)))
 for k, mol_id in enumerate(train['id'].values):
if mol_id in arnie_train_features:
arr1 = np.array(list(arnie_train_features[mol_id]['mfe']['vienna_2']))
arr2 = np.array(list(arnie_train_features[mol_id]['mfe']['contrafold_2']))
arr3 = np.array(list(arnie_train_features[mol_id]['mfe']['rnastructure']))
else:
arr1 = np.array(list(arnie_test_features[mol_id]['mfe']['vienna_2']))
arr2 = np.array(list(arnie_test_features[mol_id]['mfe']['contrafold_2']))
arr3 = np.array(list(arnie_test_features[mol_id]['mfe']['rnastructure']))
X_certainty[k, :, 0] =(( arr1 == '(')+(arr2 == '(')+(arr3 == '(')).astype(int)/ 3
X_certainty[k, :, 1] =(( arr1 == ')')+(arr2 == ')')+(arr3 == ')')).astype(int)/ 3
X_certainty[k, :, 2] =(( arr1 == '.')+(arr2 == '.')+(arr3 == '.')).astype(int)/ 3
if mol_id in capr_train_features:
X_loop_capr[k] = np.array([list(map(float, i.split() [1:])) for i in capr_train_features[mol_id].split('
')[1:-2]] ).T
else:
X_loop_capr[k] = np.array([list(map(float, i.split() [1:])) for i in capr_test_features[mol_id].split('
')[1:-2]] ).T
bpp = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).astype(np.float16)
 X_bpp_sum[k] = bpp.sum(axis=1)[:, None]
X_bpp_nb_zeroes[k] =(( bpp == 0 ).sum(axis=1)/ X_pos1.shape[1])[:, None]
X_node = np.concatenate([X_node, X_loop, X_pos1, X_pos2, X_pos3, X_certainty, X_loop_capr, X_bpp_sum, X_bpp_nb_zeroes], axis = 2)
 return X_node
X_node = get_input(train ).astype(np.float16)
 X_node_pub = get_input(test_pub ).astype(np.float16)
 X_node_pri = get_input(test_pri ).astype(np.float16 )
%%time
m_lgb = lgb.train(params, train_data, valid_sets = [fake_valid_data], verbose_eval=20 )<save_model>
del arnie_train_features, arnie_test_features, capr_train_features, capr_test_features
m_lgb.save_model("model.lgb" )<count_unique_values>
targets = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
y_train = []
seq_len = train["seq_length"].iloc[0]
seq_len_target = train["seq_scored"].iloc[0]
ignore = -10000
ignore_length = seq_len - seq_len_target
for target in targets:
y = np.vstack(train[target])
 dummy = np.zeros([y.shape[0], ignore_length])+ ignore
y = np.hstack([y, dummy])
 y_train.append(y)
 y = np.stack(y_train, axis = 2)
 print(y.shape )
sub.id.nunique() , sub["id"].str.contains("evaluation$" ).sum()<import_modules>
N_NODE_FEATURES = X_node.shape[2]
N_EDGE_FEATURES = As.shape[3]
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
import tensorflow as tf
from tensorflow.keras import layers as L
import tensorflow_addons as tfa
from tensorflow.keras import backend as K
from sklearn.utils import shuffle
from sklearn.model_selection import GroupKFold
df= pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_evaluation.csv")
 sell_prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv")
 calendar = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv" )<drop_column>
ae_epochs = 25
ae_epochs_each = 5
ae_batch_size = 32
epochs_list = [30, 10, 3, 3, 5, 5]
batch_size_list = [8, 16, 32, 64, 128, 256]
del df,store_agg,cat_agg,agg_sales,one,x
gc.collect()<set_options>
def mcrmse(t, p, seq_len_target = seq_len_target):
score = np.mean(np.sqrt(tf.keras.losses.mean_squared_error(t, p)) [:, :seq_len_target])
 return score
def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return K.sqrt(mse)
 def mcrmse_loss(y_actual, y_pred, num_scored=5):
y_actual = y_actual[:, :68, :]
y_pred = y_pred[:, :68, :]
score = 0
for i in range(num_scored):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ num_scored
return score
def attention(x_inner, x_outer, n_factor, dropout):
x_Q = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_inner)
 x_K = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_V = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_KT = L.Permute(( 2, 1))(x_K)
 res = L.Lambda(lambda c: K.batch_dot(c[0], c[1])/ np.sqrt(n_factor))([x_Q, x_KT])
 att = L.Lambda(lambda c: K.softmax(c, axis=-1))(res)
 att = L.Lambda(lambda c: K.batch_dot(c[0], c[1]))([att, x_V])
 return att
def multi_head_attention(x, y, n_factor, n_head, dropout):
if n_head == 1:
att = attention(x, y, n_factor, dropout)
 else:
n_factor_head = n_factor // n_head
heads = [attention(x, y, n_factor_head, dropout)for i in range(n_head)]
att = L.Concatenate()(heads)
 att = L.Dense(n_factor,
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(att)
 x = L.Add()([x, att])
 x = L.LayerNormalization()(x)
 if dropout > 0:
x = L.Dropout(dropout )(x)
 return x
def res(x, unit, kernel = 3, rate = 0.1):
h = L.Conv1D(unit, kernel, 1, padding = "same", activation = None )(x)
 h = L.LayerNormalization()(h)
 h = L.LeakyReLU()(h)
 h = L.Dropout(rate )(h)
 return L.Add()([x, h])
 def forward(x, unit, kernel = 3, rate = 0.1):
h = L.Conv1D(unit, kernel, 1, padding = "same", activation = None )(x)
 h = L.LayerNormalization()(h)
 h = L.Dropout(rate )(h)
 h = L.LeakyReLU()(h)
 h = res(h, unit, kernel, rate)
 return h
def adj_attn(x, adj, unit, n = 2, rate = 0.1):
x_a = x
x_as = []
for i in range(n):
x_a = forward(x_a, unit)
 x_a = tf.matmul(adj, x_a)
 x_as.append(x_a)
 if n == 1:
x_a = x_as[0]
else:
x_a = L.Concatenate()(x_as)
 x_a = forward(x_a, unit)
 return x_a
def get_base(config):
node = tf.keras.Input(shape =(None, N_NODE_FEATURES), name = "node")
 adj = tf.keras.Input(shape =(None, None, N_EDGE_FEATURES), name = "adj")
 adj_learned = L.Dense(1, "relu" )(adj)
 adj_all = L.Concatenate(axis = 3 )([adj, adj_learned])
 xs = []
xs.append(node)
 x1 = forward(node, 128, kernel = 3, rate = 0.0)
 x2 = forward(x1, 64, kernel = 6, rate = 0.0)
 x3 = forward(x2, 32, kernel = 15, rate = 0.0)
 x4 = forward(x3, 16, kernel = 30, rate = 0.0)
 x = L.Concatenate()([x1, x2, x3, x4])
 for unit in [128, 64]:
x_as = []
for i in range(adj_all.shape[3]):
x_a = adj_attn(x, adj_all[:, :, :, i], unit, rate = 0.0)
 x_as.append(x_a)
 x_c = forward(x, unit, kernel = 30)
 x = L.Concatenate()(x_as + [x_c])
 x = forward(x, unit)
 x = multi_head_attention(x, x, unit, 4, 0.0)
 xs.append(x)
 x = L.Concatenate()(xs)
 model = tf.keras.Model(inputs = [node, adj], outputs = [x])
 return model
def get_ae_model(base, config):
node = tf.keras.Input(shape =(None, N_NODE_FEATURES), name = "node")
 adj = tf.keras.Input(shape =(None, None, N_EDGE_FEATURES), name = "adj")
 x = base([L.SpatialDropout1D(0.3 )(node), adj])
 x = forward(x, 64, rate = 0.3)
 p = L.Dense(N_NODE_FEATURES, "sigmoid" )(x)
 loss = tf.reduce_mean(tf.math.abs(node - p))
model = tf.keras.Model(inputs = [node, adj], outputs = [loss])
 opt = get_optimizer()
model.compile(optimizer = opt, loss = lambda t, y : y)
 return model
def gru_layer(hidden_dim, dropout):
return L.Bidirectional(L.GRU(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def lstm_layer(hidden_dim, dropout):
return L.Bidirectional(L.LSTM(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def get_model(base, config):
node = tf.keras.Input(shape =(None, N_NODE_FEATURES), name = "node")
 adj = tf.keras.Input(shape =(None, None, N_EDGE_FEATURES), name = "adj")
 x = base([node, adj])
 x = lstm_layer(256, 0.5 )(x)
 x = lstm_layer(256, 0.5 )(x)
 x = L.Dense(5, None )(x)
 model = tf.keras.Model(inputs = [node, adj], outputs = [x])
 opt = get_optimizer()
model.compile(optimizer = opt, loss = mcrmse_loss)
 return model
def get_optimizer() :
adam = tf.optimizers.Adam()
return adam
warnings.filterwarnings('ignore' )<load_from_csv>
config = {}
if ae_epochs > 0:
base = get_base(config)
 ae_model = get_ae_model(base, config)
 for i in range(ae_epochs//ae_epochs_each):
print(f"------ {i} ------")
 print("--- train ---")
 X_node_shuff, As_shuff = shuffle(X_node, As)
 ae_model.fit([X_node_shuff, As_shuff], [np.zeros(( len(X_node)))],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 print("--- public ---")
 X_node_pub_shuff, As_pub_shuff = shuffle(X_node_pub, As_pub)
 ae_model.fit([X_node_pub_shuff, As_pub_shuff], [np.zeros(( len(X_node_pub)))],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 print("--- private ---")
 X_node_pri_shuff, As_pri_shuff = shuffle(X_node_pri, As_pri)
 ae_model.fit([X_node_pri_shuff, As_pri_shuff], [np.zeros(( len(X_node_pri)))],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 gc.collect()
print("****** save ae model ******")
 base.save_weights("base_ae_lstm_lstm" )
calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv')
 calendar = reduce_mem_usage(calendar)
 sell_prices = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sell_prices.csv')
 sell_prices = reduce_mem_usage(sell_prices)
 sales_train_validation = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 submission = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sample_submission.csv')
 submission = reduce_mem_usage(submission )<categorify>
del ae_model
gc.collect()
sales_train_validation = pd.melt(sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name = 'day',
value_name = 'demand')
 sales_train_validation = reduce_mem_usage(sales_train_validation )<drop_column>
np.save('X_node_pub.npy', X_node_pub)
 np.save('X_node_pri.npy', X_node_pri)
 np.save('As_pub.npy', As_pub)
 np.save('As_pri.npy', As_pri)
 del X_node_pub, X_node_pri, As_pub, As_pri, X_node_shuff, As_shuff, X_node_pub_shuff, As_pub_shuff, X_node_pri_shuff, As_pri_shuff
test2_rows = [row for row in submission['id'] if 'evaluation' in row]
test2 = submission[submission['id'].isin(test2_rows)]
test2.columns = ['id', 'd_1942', 'd_1943', 'd_1944', 'd_1945', 'd_1946', 'd_1947', 'd_1948', 'd_1949', 'd_1950', 'd_1951', 'd_1952', 'd_1953', 'd_1954', 'd_1955', 'd_1956', 'd_1957', 'd_1958', 'd_1959',
'd_1960', 'd_1961', 'd_1962', 'd_1963', 'd_1964', 'd_1965', 'd_1966', 'd_1967', 'd_1968', 'd_1969']
product = sales_train_validation[['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']].drop_duplicates()
test2 = test2.merge(product, how = 'left', on = 'id')
 test2 = pd.melt(test2,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name = 'day',
value_name = 'demand')
 sales_train_validation['part'] = 'train'
test2['part'] = 'test'<concatenate>
kfold = GroupKFold(5)
 config = {}
scores = []
preds = np.zeros([len(X_node), X_node.shape[1], 5])
 X_node, As, y, weights, groups_shuffled, SN_filter_mask = shuffle(X_node, As, y, train.signal_to_noise.values, train['id'],(train['SN_filter'] == 1 ).values)
 del train
for i,(tr_idx, va_idx)in enumerate(kfold.split(X_node, As, groups_shuffled)) :
print(f"------ fold {i} start -----")
 X_node_tr = X_node[tr_idx]
X_node_va = X_node[va_idx]
As_tr = As[tr_idx]
As_va = As[va_idx]
y_tr = y[tr_idx]
y_va = y[va_idx]
w_trn = np.log(weights[tr_idx]+1.105)/2
base = get_base(config)
 if ae_epochs > 0:
print("****** load ae model ******")
 base.load_weights("base_ae_lstm_lstm")
 model = get_model(base, config)
 for epochs, batch_size in zip(epochs_list, batch_size_list):
print(f"epochs : {epochs}, batch_size : {batch_size}")
 model.fit([X_node_tr, As_tr], [y_tr],
validation_data=([X_node_va[SN_filter_mask[va_idx]], As_va[SN_filter_mask[va_idx]]], [y_va[SN_filter_mask[va_idx]]]),
epochs = epochs,
sample_weight=w_trn/2,
batch_size = batch_size, validation_freq = 3)
 model.save_weights(f"model{i}_lstm_lstm")
 p = model.predict([X_node_va, As_va])
 for col in range(5):
print(targets[col],(( p[:, :68, col][SN_filter_mask[va_idx]] - y_va[:,:68,col][SN_filter_mask[va_idx]])** 2 ).mean() **.5)
 scores.append(np.mean(mcrmse_loss(y_va, p)))
 print(f"fold {i}: mcrmse {scores[-1]}")
 preds[va_idx] = p
gc.collect()
data = pd.concat([sales_train_validation,test2], axis = 0)
 del sales_train_validation,test2
gc.collect()<merge>
preds_df = []
for p_ix, _id in zip(range(preds.shape[0]), groups_shuffled):
for i in range(68):
preds_df.append([f'{_id}_{i}', preds[p_ix, i, 0], preds[p_ix, i, 1],
preds[p_ix, i, 3], y[p_ix, i, 0], y[p_ix, i, 1],
y[p_ix, i, 3], SN_filter_mask[p_ix]])
 preds_df = pd.DataFrame(preds_df, columns=['id_seqpos', 'reactivity', 'deg_Mg_pH10',
'deg_Mg_50C', 'reactivity_gt', 'deg_Mg_pH10_gt',
'deg_Mg_50C_gt', 'SN_filter'])
 preds_df = preds_df.groupby('id_seqpos' ).mean()
rmses = []
mses = []
for col in ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']:
_rmse =(( preds_df[preds_df['SN_filter'] == 1][col] - preds_df[preds_df['SN_filter'] == 1][col+'_gt'])** 2 ).mean() **.5
_mse =(( preds_df[preds_df['SN_filter'] == 1][col] - preds_df[preds_df['SN_filter'] == 1][col+'_gt'])** 2 ).mean()
rmses.append(_rmse)
 mses.append(_mse)
 print(col, _rmse, _mse)
 print(np.mean(rmses), np.mean(mses))
data.reset_index(drop=True,inplace=True)
 data = data.loc[30000000:]
calendar.drop(['weekday', 'wday', 'month', 'year'], inplace = True, axis = 1)
 data = pd.merge(data, calendar, how = 'left', left_on = ['day'], right_on = ['d'])
 data.drop(['d', 'day'], inplace = True, axis = 1 )<merge>
del X_node, As
data = data.merge(sell_prices, on = ['store_id', 'item_id', 'wm_yr_wk'], how = 'left')
 del calendar,sell_prices,product
gc.collect()<categorify>
X_node_pub = np.load('X_node_pub.npy')
 X_node_pri = np.load('X_node_pri.npy')
 As_pub = np.load('As_pub.npy')
 As_pri = np.load('As_pri.npy' )
nan_features = ['event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for feature in nan_features:
data[feature].fillna('unknown', inplace = True)
 cat = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for feature in cat:
encoder = LabelEncoder()
data[feature] = encoder.fit_transform(data[feature] )<statistical_test>
p_pub = 0
p_pri = 0
for i in range(5):
config = {}
base = get_base(config)
 if ae_epochs > 0:
print("****** load ae model ******")
 base.load_weights("base_ae_lstm_lstm")
 model = get_model(base, config)
 model.load_weights(f"model{i}_lstm_lstm")
 p_pub += model.predict([X_node_pub, As_pub])/ 5
p_pri += model.predict([X_node_pri, As_pri])/ 5
for i, target in enumerate(targets):
test_pub[target] = [list(p_pub[k, :, i])for k in range(p_pub.shape[0])]
test_pri[target] = [list(p_pri[k, :, i])for k in range(p_pri.shape[0])]
<feature_engineering><EOS> 
preds_ls = []
for df, preds in [(test_pub, p_pub),(test_pri, p_pri)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=targets)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls)
 preds_df = preds_df.groupby('id_seqpos' ).mean()
preds_df.to_csv("submission_lstm_lstm.csv")
 preds_df.head()