Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<feature_engineering>
warnings.filterwarnings("ignore" )
lockdown_dates_upd['Lockdown_date_use'] = ''
lockdown_dates_upd['Lockdown_date_use'][lockdown_dates_upd['Country_Region'].isin(['US','China','Australia'])== True] = lockdown_dates_upd['Lockdown_date']
lockdown_dates_upd['Lockdown_date_use'][lockdown_dates_upd['Country_Region'].isin(['US','China','Australia'])== False] = lockdown_dates_upd['Stringency_Date'].combine_first(lockdown_dates_upd['Lockdown_date'])
<feature_engineering>
seed = 42
def cc_model_poly(dataset,start_date,end_date,start_value,Country_State,knot_number):
dataset = dataset[(dataset['Date'] >= start_date)&(dataset['Date'] <= end_date)]
dataset['Date_diff'] =(((pd.to_datetime(dataset['Date'])- pd.to_datetime(start_date)).dt.days))
dataset['Date_diff_p2'] = dataset['Date_diff']**2
dataset['Date_diff_p3'] = dataset['Date_diff']**3
input_columns = ['Date_diff','Date_diff_p2','Date_diff_p3']
if len(dataset.dropna())> 5:
start_value = dataset['ConfirmedCases_inc'].values[0]
dataset['ConfirmedCases_inc_use'] = dataset['ConfirmedCases_inc_sm']
reg = LinearRegression().fit(dataset.dropna() [input_columns], dataset.dropna() ['ConfirmedCases_inc_use'])
 dataset['ConfirmedCases_pred'] = reg.predict(dataset[input_columns])
 filename = 'cc_'+Country_State+'_'+knot_number+'.sav'
pickle.dump(reg, open(filename, 'wb'))
else:
print('.. picking fallback cc model for knot '+knot_number+'.. ')
 reg = pickle.load(open('cc_China Hubei'+'_'+knot_number+'.sav', 'rb'))
dataset['ConfirmedCases_pred'] = reg.predict(dataset[input_columns])- reg.intercept_ + start_value
if knot_number == '2':
cap_upper = dataset['ConfirmedCases_inc'].mean() +(dataset['ConfirmedCases_inc'].std())
 cap_lower = dataset['ConfirmedCases_inc'].mean() -(dataset['ConfirmedCases_inc'].std())
 dataset['ConfirmedCases_pred'] = dataset['ConfirmedCases_pred'].clip(lower=cap_lower, upper=cap_upper)
 dataset['ConfirmedCases_pred'][dataset['ConfirmedCases_pred'] < 0] = 0
return(dataset )<feature_engineering>
DEVICE = "TPU"
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 print('Number of devices: {}'.format(strategy.num_replicas_in_sync))
def cc_model_exp(dataset,start_date,end_date,start_value,Country_State,knot_number):
dataset = dataset[(dataset['Date'] >= start_date)&(dataset['Date'] <= end_date)]
dataset['Date_diff'] =(((pd.to_datetime(dataset['Date'])- pd.to_datetime(start_date)).dt.days))
if len(dataset.dropna())> 5:
dataset['ConfirmedCases_inc_use'] = dataset['ConfirmedCases_inc_sm'] + 0.0001
reg = LinearRegression().fit(dataset.dropna() [['Date_diff']], np.log(dataset.dropna() ['ConfirmedCases_inc_use']))
dataset['ConfirmedCases_pred'] = np.exp(reg.predict(dataset[['Date_diff']]))
filename = 'cc_'+Country_State+'_'+knot_number+'.sav'
pickle.dump(reg, open(filename, 'wb'))
else:
print('.. picking fallback cc model for knot '+knot_number+'.. ')
 reg = pickle.load(open('cc_China Hubei'+'_'+knot_number+'.sav', 'rb'))
dataset['ConfirmedCases_pred'] = np.exp(reg.predict(dataset[['Date_diff']])- reg.intercept_ + np.log(start_value))
if reg.coef_ > 0:
print('.. skipping exp model as the curve is not declining for knot '+knot_number+'.. ')
 dataset['ConfirmedCases_pred'] = dataset['ConfirmedCases_inc'][dataset['ConfirmedCases'] > 0].mean()
return(dataset )<choose_model_class>
dropout_model = 0.36
hidden_dim_first = 128
hidden_dim_second = 256
hidden_dim_third = 128
def fa_model_lnr(dataset,Country_State):
dataset['ConfirmedCases_lagn'] = dataset['ConfirmedCases'].shift(periods=5)
 dataset['ConfirmedCases_pred_lagn'] = dataset['ConfirmedCases_pred'].shift(periods=5 ).fillna(0)
 if len(dataset.dropna())> 10:
reg = LinearRegression(fit_intercept=False ).fit(dataset.dropna() [['ConfirmedCases_lagn']], dataset.dropna() ['Fatalities'])
 dataset['Fatalities_pred'] = reg.predict(dataset[['ConfirmedCases_pred_lagn']])
 filename = 'fa_'+Country_State+'_'+'.sav'
pickle.dump(reg, open(filename, 'wb'))
else:
print('.. picking fallback fa model.. ')
 loaded_model = pickle.load(open('fa_China Hubei'+'_'+'.sav', 'rb'))
dataset['Fatalities_pred'] = loaded_model.predict(dataset[['ConfirmedCases_pred_lagn']])
 return(dataset )<feature_engineering>
train = pd.read_json('/kaggle/input/stanford-covid-vaccine/train.json', lines=True)
 print("train shape",train.shape)
 test = pd.read_json('/kaggle/input/stanford-covid-vaccine/test.json', lines=True)
 sample_sub = pd.read_csv("/kaggle/input/stanford-covid-vaccine/sample_submission.csv")
test_update = pd.DataFrame(columns = ['Province_State', 'Country_Region', 'Date', 'ConfirmedCases_pred' ,'Fatalities_pred'])
 for Country_State in ['China Hubei'] + master['Country_State'].unique().tolist() :
print('running for: '+Country_State)
 master_use = master[master['Country_State'] == Country_State]
master_use['ConfirmedCases_inc'] = master_use['ConfirmedCases'] - master_use['ConfirmedCases'].shift(periods=1 ).fillna(0)
 master_use['ConfirmedCases_inc_sm'] = master_use['ConfirmedCases_inc'].rolling(window=5,center=True,min_periods=1 ).mean()
st_dt = master_use['Date'][master_use['ConfirmedCases'] > 0].min()
mx_dt = master_use['Date'].max()
ld_dt = lockdown_dates_upd['Lockdown_date_use'][lockdown_dates_upd['Country_State'] == Country_State].values[0]
try:
if(ld_dt != '9999-12-31'):
add_wdw = 3
kt_1p =(pd.to_datetime(ld_dt)+dt.timedelta(days=14+add_wdw)).strftime("%Y-%m-%d")
 kt_1m =(pd.to_datetime(ld_dt)+dt.timedelta(days=14-add_wdw)).strftime("%Y-%m-%d")
 kt_2p =(pd.to_datetime(ld_dt)+dt.timedelta(days=28+add_wdw)).strftime("%Y-%m-%d")
 kt_2m =(pd.to_datetime(ld_dt)+dt.timedelta(days=28-add_wdw)).strftime("%Y-%m-%d")
 a = cc_model_poly(master_use, st_dt, kt_1p, 0, Country_State, '1')
 b = cc_model_poly(master_use, kt_1m, kt_2p, a.tail(1)['ConfirmedCases_pred'].values[0], Country_State, '2')
 c = cc_model_exp(master_use, kt_2m, mx_dt, b.tail(1)['ConfirmedCases_pred'].values[0], Country_State, '3')
 master_use = a.append(b ).append(c)
 master_use = master_use.groupby(['Province_State', 'Country_Region', 'Date', 'Country_State'])['ConfirmedCases', 'ConfirmedCases_pred', 'ConfirmedCases_inc', 'Fatalities'].mean().reset_index()
else:
master_use = cc_model_poly(master_use, st_dt, mx_dt, 0, Country_State, '1')
 master_use['ConfirmedCases_pred'] = master_use['ConfirmedCases_inc'].combine_first(master_use['ConfirmedCases_pred'])
 master_use['ConfirmedCases_pred'] = master_use['ConfirmedCases_pred'].cumsum()
master_use = fa_model_lnr(master_use, Country_State)
 master_use['Fatalities_pred'] = master_use['Fatalities'].combine_first(master_use['Fatalities_pred'])
 test_use = master_use[master_use['Date'] >= test_date_st]
test_update = test_update.append(test_use[['Province_State', 'Country_Region', 'Date', 'ConfirmedCases_pred', 'Fatalities_pred']])
 except:
print('exiting due to error')
 pass<feature_engineering>
def read_bpps_sum(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).max(axis=1))
return bpps_arr
def read_bpps_max(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).sum(axis=1))
return bpps_arr
def read_bpps_nb(df):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps_arr = []
for mol_id in df.id.to_list() :
bpps = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy")
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
bpps_arr.append(bpps_nb)
 return bpps_arr
for Country_State in missing_inst:
print('running for: '+Country_State)
 master_use = master[master['Country_State'] == Country_State]
cc_ext = master_use['ConfirmedCases'].max()
fa_ext = master_use['Fatalities'].max()
master_use['ConfirmedCases_pred'] = cc_ext
master_use['Fatalities_pred'] = fa_ext
master_use['ConfirmedCases_pred'] = master_use['ConfirmedCases'].combine_first(master_use['ConfirmedCases_pred'])
 test_use = master_use[master_use['Date'] >= test_date_st]
test_update = test_update.append(test_use[['Province_State', 'Country_Region', 'Date', 'ConfirmedCases_pred', 'Fatalities_pred']] )<groupby>
train['bpps_sum'] = read_bpps_sum(train)
 test['bpps_sum'] = read_bpps_sum(test)
 train['bpps_max'] = read_bpps_max(train)
 test['bpps_max'] = read_bpps_max(test)
 train['bpps_nb'] = read_bpps_nb(train)
 test['bpps_nb'] = read_bpps_nb(test )
test_update = test_update.groupby(['Province_State', 'Country_Region', 'Date'])['ConfirmedCases_pred','Fatalities_pred'].max().reset_index()<load_from_csv>
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
def preprocess_inputs(df, cols=['sequence','predicted_loop_type','structure']):
base_fea = np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 bpps_sum_fea = np.array(df['bpps_sum'].to_list())[:,:,np.newaxis]
bpps_max_fea = np.array(df['bpps_max'].to_list())[:,:,np.newaxis]
bpps_nb_fea = np.array(df['bpps_nb'].to_list())[:,:,np.newaxis]
_, position_fea = np.mgrid[0:bpps_nb_fea.shape[0]:1, 0:bpps_nb_fea.shape[1]:1]/(bpps_nb_fea.shape[1]-1)
 return np.concatenate([base_fea,bpps_sum_fea,bpps_max_fea,bpps_nb_fea, position_fea[:,:,np.newaxis]], 2)
 kmeans_model = KMeans(n_clusters=200, random_state=110 ).fit(preprocess_inputs(train)[:,:,0])
 train['cluster_id'] = kmeans_model.labels_
test_orig = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 test_orig['Province_State'] = test_orig['Province_State'].fillna('' )<merge>
aug_df = pd.read_csv('.. /input/augmented-data-for-stanford-covid-vaccine/48k_augment.csv')
 display(aug_df.head() )
sub = pd.merge(test_orig[['ForecastId','Province_State','Country_Region', 'Date']], test_update[['Province_State','Country_Region', 'Date', 'ConfirmedCases_pred','Fatalities_pred']], how='left', on=['Province_State','Country_Region', 'Date'] )<data_type_conversions>
def aug_data(df):
target_df = df.copy()
new_df = aug_df[aug_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id','sequence'], how='left')
 df['cnt'] = df['id'].map(new_df[['id','cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
df = df.append(new_df[df.columns])
 return df
train = aug_data(train)
 test = aug_data(test )
sub['ConfirmedCases_pred'] = sub['ConfirmedCases_pred'].fillna(0)
 sub['Fatalities_pred'] = sub['Fatalities_pred'].fillna(0 )<prepare_output>
train_inputs_all = preprocess_inputs(train)
 train_labels_all = np.array(train[target_cols].values.tolist() ).transpose(( 0, 2, 1))
sub = sub[['ForecastId','ConfirmedCases_pred','Fatalities_pred']]<save_to_csv>
def scaled_dot_product_attention(q, k, v, mask):

 matmul_qk = tf.matmul(q, k, transpose_b=True)
 dk = tf.cast(tf.shape(k)[-1], tf.float32)
 scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
 if mask is not None:
scaled_attention_logits +=(mask * -1e9)
 attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)
 output = tf.matmul(attention_weights, v)
 return output, attention_weights
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads):
super(MultiHeadAttention, self ).__init__()
self.num_heads = num_heads
self.d_model = d_model
assert d_model % self.num_heads == 0
self.depth = d_model // self.num_heads
self.wq = tf.keras.layers.Dense(d_model)
 self.wk = tf.keras.layers.Dense(d_model)
 self.wv = tf.keras.layers.Dense(d_model)
 self.dense = tf.keras.layers.Dense(d_model)
 def split_heads(self, x, batch_size):

 x = tf.reshape(x,(batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
 def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.wq(q)
 k = self.wk(k)
 v = self.wv(v)
 q = self.split_heads(q, batch_size)
 k = self.split_heads(k, batch_size)
 v = self.split_heads(v, batch_size)
 scaled_attention, attention_weights = scaled_dot_product_attention(
 q, k, v, mask)
 scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])
 concat_attention = tf.reshape(scaled_attention, 
(batch_size, -1, self.d_model))
output = self.dense(concat_attention)
 return output, attention_weights
def get_config(self):
config = super().get_config().copy()
config.update({
'depth': self.depth,
'wq': self.wq,
'qk': self.wk,
'wv': self.wv,
'dense': self.dense,
})
 return config
def point_wise_feed_forward_network(d_model, dff):
return tf.keras.Sequential([
tf.keras.layers.Dense(dff, activation='relu'),
tf.keras.layers.Dense(d_model)
 ])
 class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.1):
super(EncoderLayer, self ).__init__()
self.d_model = d_model
self.num_heads = num_heads
self.dff = dff
self.rate = rate
self.mha = MultiHeadAttention(d_model, num_heads)
 self.ffn = point_wise_feed_forward_network(d_model, dff)
 self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.dropout1 = tf.keras.layers.Dropout(rate)
 self.dropout2 = tf.keras.layers.Dropout(rate)
 def call(self, x, training):
attn_output, _ = self.mha(x, x, x, None)
 attn_output = self.dropout1(attn_output, training=training)
 out1 = self.layernorm1(x + attn_output)
 ffn_output = self.ffn(out1)
 ffn_output = self.dropout2(ffn_output, training=training)
 out2 = self.layernorm2(out1 + ffn_output)
 return out2
def get_config(self):
config = super().get_config().copy()
config.update({
'num_heads': self.num_heads,
'rate': self.rate,
'd_model': self.d_model,
'num_heads': self.num_heads,
'dropout1': self.dropout1,
'dropout2': self.dropout2,
'layernorm1': self.layernorm1,
'layernorm2': self.layernorm2,
'mha': self.mha,
'ffn': self.ffn,
})
 return config
def gru_layer(hidden_dim, dropout):
return L.Bidirectional(L.GRU(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def lstm_layer(hidden_dim, dropout):
return L.Bidirectional(L.LSTM(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def MCRMSE(y_true, y_pred):
colwise_mse = tf.reduce_mean(tf.square(y_true - y_pred), axis=1)
 return tf.reduce_mean(tf.sqrt(colwise_mse), axis=1 )
sub.to_csv("submission.csv",index=False )<drop_column>
def build_model(model_type=1, seq_len=107, pred_len=68, embed_dim=200,
dropout=dropout_model, hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second, hidden_dim_third = hidden_dim_third):
inputs = tf.keras.layers.Input(shape=(seq_len, 7))
categorical_feat_dim = 3
categorical_fea = inputs[:, :, :categorical_feat_dim]
numerical_fea = inputs[:, :, 3:6]
positional_fea = tf.expand_dims(inputs[:, :, 6], axis=2)
 embed = tf.keras.layers.Embedding(input_dim=len(token2int), output_dim=embed_dim )(categorical_fea)
 reshaped = tf.reshape(
 embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = L.concatenate([reshaped, numerical_fea], axis=2)
 reshaped = tf.keras.layers.SpatialDropout1D (.2 )(reshaped)
 conv = L.Conv1D(255, 3, padding='same', activation=tf.keras.activations.swish )(reshaped)
 conv = L.Conv1D(255, 6, padding='same', activation=tf.keras.activations.swish )(conv)
 reshaped = L.concatenate([conv, positional_fea], axis=2)
 hidden = EncoderLayer(256, 128, 512 )(reshaped)
 hidden = EncoderLayer(256, 128, 512 )(hidden)
 if model_type == 0:
hidden = gru_layer(256, 0.3 )(hidden)
 hidden = gru_layer(256, 0.3 )(hidden)
 elif model_type == 1:
hidden = lstm_layer(256, 0.3 )(hidden)
 hidden = gru_layer(256, 0.3 )(hidden)
 elif model_type == 2:
hidden = gru_layer(256, 0.3 )(hidden)
 hidden = lstm_layer(256, 0.3 )(hidden)
 elif model_type == 3:
hidden = lstm_layer(256, 0.3 )(hidden)
 hidden = lstm_layer(256, 0.3 )(hidden)
 truncated = hidden[:, :pred_len]
out = tf.keras.layers.Dense(len(target_cols), activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 adam = tf.optimizers.Adam()
model.compile(optimizer=adam, loss=MCRMSE, metrics=[tf.keras.metrics.RootMeanSquaredError() ])
 return model
for dirname, _, filenames in os.walk('.. '):
for filename in filenames:
if filename.lower().endswith('.sav'):
os.remove(os.path.join(dirname, filename))<load_from_csv>
tf.keras.backend.clear_session()
lr_callback = tf.keras.callbacks.ReduceLROnPlateau()
es_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss',restore_best_weights=True,min_delta=0.001, patience=10 )
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 train['CRPS']=train.Country_Region+train.Province_State.fillna('')
 test['CRPS']=test.Country_Region+test.Province_State.fillna('')
 train<feature_engineering>
mse_s = []
rmse_s = []
def train_and_predict(n_folds=5, model_name="model", model_type=0, epochs=100, debug=True,
dropout_model=dropout_model, hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second, hidden_dim_third = hidden_dim_third,
seed=seed):
print("Model:", model_name)
 ensemble_preds = pd.DataFrame(index=sample_sub.index, columns=target_cols ).fillna(0)
 kf = KFold(n_folds, shuffle=True, random_state=seed)
 skf = StratifiedKFold(n_folds, shuffle=True, random_state=seed)
 gkf = GroupKFold(n_splits=n_folds)
 val_losses = []
historys = []
for i,(train_index, val_index)in enumerate(gkf.split(train, train['reactivity'], train['cluster_id'])) :
print("Fold:", str(i+1))
with strategy.scope() :
model_train = build_model(model_type=model_type,
dropout=dropout_model,
hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second,
hidden_dim_third = hidden_dim_third)
 model_short = build_model(model_type=model_type, seq_len=107, pred_len=107,
dropout=dropout_model,
hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second,
hidden_dim_third = hidden_dim_third)
 model_long = build_model(model_type=model_type, seq_len=130, pred_len=130,
dropout=dropout_model,
hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second,
hidden_dim_third = hidden_dim_third)
 train_inputs, train_labels = train_inputs_all[train_index], train_labels_all[train_index]
val = train.iloc[val_index]
x_val_all = preprocess_inputs(val)
 val = val[val.SN_filter == 1]
val_inputs = preprocess_inputs(val)
 val_labels = np.array(val[target_cols].values.tolist() ).transpose(( 0, 2, 1))
w_trn = np.log(train.iloc[train_index].signal_to_noise+1.1)/2
checkpoint = tf.keras.callbacks.ModelCheckpoint(f'{model_name}_Fold_{str(i+1)}.h5')
 history = model_train.fit(
 train_inputs , train_labels,
validation_data=(val_inputs,val_labels),
batch_size=64,
sample_weight=w_trn/2,
epochs=epochs,
callbacks=[checkpoint,
lr_callback,
TqdmCallback() ,
tf.keras.callbacks.TerminateOnNaN() ,
es_callback],
verbose= 0 
)
 holdouts = train.iloc[val_index]
holdout_preds = model_train.predict(x_val_all)
 holdout_labels = np.array(holdouts[target_cols].values.tolist() ).transpose(( 0, 2, 1))
rmse =(( holdout_labels - holdout_preds)** 2 ).mean() **.5
mse =(( holdout_labels - holdout_preds)** 2 ).mean()
print(f"{model_name} Min training loss={min(history.history['loss'])}, min validation loss={min(history.history['val_loss'])}")
 print(f"{model_name} Holdouts mse ={mse}, Holdouts rmse ={rmse}")
 mse_s.append(mse)
 rmse_s.append(rmse)
 mse_s_t.append(mse)
 rmse_s_t.append(rmse)
 val_losses.append(min(history.history['val_loss']))
historys.append(history)
 model_short.load_weights(f'{model_name}_Fold_{str(i+1)}.h5')
 model_long.load_weights(f'{model_name}_Fold_{str(i+1)}.h5')
 public_preds = model_short.predict(public_inputs)
 private_preds = model_long.predict(private_inputs)
 preds_model = []
for df, preds in [(public_df, public_preds),(private_df, private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_model.append(single_df)
 preds_model_df = pd.concat(preds_model ).groupby('id_seqpos' ).mean().reset_index()
ensemble_preds[target_cols] += preds_model_df[target_cols].values / n_folds
if debug:
print("Intermediate ensemble result")
 print(ensemble_preds[target_cols].head())
 ensemble_preds["id_seqpos"] = preds_model_df["id_seqpos"].values
ensemble_preds = pd.merge(sample_sub["id_seqpos"], ensemble_preds, on="id_seqpos", how="left")
 print("Mean Validation loss:", str(np.mean(val_losses)))
 if debug:
fig, ax = plt.subplots(1, 3, figsize =(20, 10))
for i, history in enumerate(historys):
ax[0].plot(history.history['loss'])
 ax[0].plot(history.history['val_loss'])
 ax[0].set_title('model_'+str(i+1))
ax[0].set_ylabel('Loss')
 ax[0].set_xlabel('Epoch')
 ax[1].plot(history.history['root_mean_squared_error'])
 ax[1].plot(history.history['val_root_mean_squared_error'])
 ax[1].set_title('model_'+str(i+1))
ax[1].set_ylabel('RMSE')
 ax[1].set_xlabel('Epoch')
 ax[2].plot(history.history['lr'])
 ax[2].set_title('model_'+str(i+1))
ax[2].set_ylabel('LR')
 ax[2].set_xlabel('Epoch')
 plt.show()
return ensemble_preds
public_df = test.query("seq_length == 107" ).copy()
private_df = test.query("seq_length == 130" ).copy()
public_inputs = preprocess_inputs(public_df)
 private_inputs = preprocess_inputs(private_df)
 ensembles = []
for i in range(1):
model_name = "model_"+str(i+1)
 mse_s_t = []
rmse_s_t = []
ensemble = train_and_predict(n_folds=6, model_name=model_name, model_type=i, epochs=70,
dropout_model=dropout_model, hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second, hidden_dim_third = hidden_dim_third,
seed=seed)
 ensembles.append(ensemble)
 print("RMSE Avg ", np.array(rmse_s_t ).mean())
 print("MSE Avg ", np.array(mse_s_t ).mean() )
train['LConfirmedCases']=np.log1p(train['ConfirmedCases'])
 train['LFatalities']=np.log1p(train['Fatalities'])
 train['LDConfirmedCases']=train.groupby('CRPS')[['LConfirmedCases']].diff()
train['LDFatalities']=train.groupby('CRPS')[['LFatalities']].diff()
train['LConfirmedCases1']=train.groupby('CRPS')[['LConfirmedCases']].shift(1)
 train['LConfirmedCases2']=train.groupby('CRPS')[['LConfirmedCases']].shift(2)
 train['LConfirmedCases3']=train.groupby('CRPS')[['LConfirmedCases']].shift(3)
 train['LConfirmedCases4']=train.groupby('CRPS')[['LConfirmedCases']].shift(4)
 train['LConfirmedCases5']=train.groupby('CRPS')[['LConfirmedCases']].shift(5)
 train['LDConfirmedCases1']=train.groupby('CRPS')[['LDConfirmedCases']].shift(1)
 train['LDConfirmedCases2']=train.groupby('CRPS')[['LDConfirmedCases']].shift(2)
 train['LDConfirmedCases3']=train.groupby('CRPS')[['LDConfirmedCases']].shift(3)
 train['LDConfirmedCases4']=train.groupby('CRPS')[['LDConfirmedCases']].shift(4)
 train['LDConfirmedCases5']=train.groupby('CRPS')[['LDConfirmedCases']].shift(5)
 train['LDConfirmedCasesMA']=(train.LDConfirmedCases+train.LDConfirmedCases1+train.LDConfirmedCases2+train.LDConfirmedCases3+train.LDConfirmedCases4+train.LDConfirmedCases5)/6
train['LDConfirmedCasesMA1']=train.groupby('CRPS')[['LDConfirmedCasesMA']].shift(1)
 train['LDConfirmedCasesMA2']=train.groupby('CRPS')[['LDConfirmedCasesMA']].shift(2)
 train['LDConfirmedCasesMA3']=train.groupby('CRPS')[['LDConfirmedCasesMA']].shift(3)
 train['LDConfirmedCasesMA4']=train.groupby('CRPS')[['LDConfirmedCasesMA']].shift(4)
 train['LDConfirmedCasesMA5']=train.groupby('CRPS')[['LDConfirmedCasesMA']].shift(5)
 train['LConfirmedCasesMA']=(train.LConfirmedCases+train.LConfirmedCases1+train.LConfirmedCases2+train.LConfirmedCases3+train.LConfirmedCases4+train.LConfirmedCases5)/6
train['LConfirmedCasesMA1']=train.groupby('CRPS')[['LConfirmedCasesMA']].shift(1)
 train['LConfirmedCasesMA2']=train.groupby('CRPS')[['LConfirmedCasesMA']].shift(2)
 train['LConfirmedCasesMA3']=train.groupby('CRPS')[['LConfirmedCasesMA']].shift(3)
 train['LConfirmedCasesMA4']=train.groupby('CRPS')[['LConfirmedCasesMA']].shift(4)
 train['LConfirmedCasesMA5']=train.groupby('CRPS')[['LConfirmedCasesMA']].shift(5)
 train['LFatalities1']=train.groupby('CRPS')[['LFatalities']].shift(1)
 train['LFatalities2']=train.groupby('CRPS')[['LFatalities']].shift(2)
 train['LFatalities3']=train.groupby('CRPS')[['LFatalities']].shift(3)
 train['LFatalities4']=train.groupby('CRPS')[['LFatalities']].shift(4)
 train['LFatalities5']=train.groupby('CRPS')[['LFatalities']].shift(5)
 train['LDFatalities1']=train.groupby('CRPS')[['LDFatalities']].shift(1)
 train['LDFatalities2']=train.groupby('CRPS')[['LDFatalities']].shift(2)
 train['LDFatalities3']=train.groupby('CRPS')[['LDFatalities']].shift(3)
 train['LDFatalities4']=train.groupby('CRPS')[['LDFatalities']].shift(4)
 train['LDFatalities5']=train.groupby('CRPS')[['LDFatalities']].shift(5)
 train['LDFatalitiesMA']=(train.LDFatalities+train.LDFatalities1+train.LDFatalities2+train.LDFatalities3+train.LDFatalities4+train.LDFatalities5)/6
train['LDFatalitiesMA1']=train.groupby('CRPS')[['LDFatalitiesMA']].shift(1)
 train['LDFatalitiesMA2']=train.groupby('CRPS')[['LDFatalitiesMA']].shift(2)
 train['LDFatalitiesMA3']=train.groupby('CRPS')[['LDFatalitiesMA']].shift(3)
 train['LDFatalitiesMA4']=train.groupby('CRPS')[['LDFatalitiesMA']].shift(4)
 train['LDFatalitiesMA5']=train.groupby('CRPS')[['LDFatalitiesMA']].shift(5)
 train['LFatalitiesMA']=(train.LFatalities+train.LFatalities1+train.LFatalities2+train.LFatalities3+train.LFatalities4+train.LFatalities5)/6
train['LFatalitiesMA1']=train.groupby('CRPS')[['LFatalitiesMA']].shift(1)
 train['LFatalitiesMA2']=train.groupby('CRPS')[['LFatalitiesMA']].shift(2)
 train['LFatalitiesMA3']=train.groupby('CRPS')[['LFatalitiesMA']].shift(3)
 train['LFatalitiesMA4']=train.groupby('CRPS')[['LFatalitiesMA']].shift(4)
 train['LFatalitiesMA5']=train.groupby('CRPS')[['LFatalitiesMA']].shift(5)
 train['serd']=train.groupby('CRPS' ).cumcount()
train.loc[train.ConfirmedCases==0,'days_since_confirmed']=0
train.loc[train.ConfirmedCases>0,'days_since_confirmed']=train[train.ConfirmedCases>0].groupby('CRPS' ).cumcount() 
<choose_model_class>
ensemble_final = ensembles[0].copy()
ensemble_final[target_cols] = 0
for ensemble in ensembles:
ensemble_final[target_cols] += ensemble[target_cols].values / len(ensembles)
 ensemble_final.head().T
lgbm_cc=LGBMRegressor(num_leaves = 85,learning_rate =10**-1.89,n_estimators=100,min_sum_hessian_in_leaf=(10**-4.1),min_child_samples =2,subsample =0.97,subsample_freq=10,
colsample_bytree = 0.68,reg_lambda=10**1.4,random_state=1234,n_jobs=4)
 lgbm_f=LGBMRegressor(num_leaves = 26,learning_rate =10**-1.63,n_estimators=100,min_sum_hessian_in_leaf=(10**-4.04),min_child_samples =14,subsample =0.66,subsample_freq=5,
colsample_bytree = 0.8,reg_lambda=10**1.92,random_state=1234,n_jobs=4)
 oe=OrdinalEncoder()
X=oe.fit_transform(train[['Country_Region','Province_State']].fillna(''))
train['CR']=X[:,0]
train['PS']=X[:,1]
lgbm_cc.fit(train.loc[:,['LDConfirmedCases1','LDConfirmedCases2','LDConfirmedCases3','LDConfirmedCases4','LDConfirmedCases5',
'LDFatalities1','LDFatalities2','LDFatalities3','LDFatalities4','LDFatalities5','days_since_confirmed','CR','PS',
'LDConfirmedCasesMA1','LDConfirmedCasesMA2','LDConfirmedCasesMA3','LDConfirmedCasesMA4','LDConfirmedCasesMA5',
'LDFatalitiesMA1','LDFatalitiesMA2','LDFatalitiesMA3','LDFatalitiesMA4','LDFatalitiesMA5',
'LConfirmedCases1','LConfirmedCases2','LConfirmedCases3','LConfirmedCases4','LConfirmedCases5',
'LFatalities1','LFatalities2','LFatalities3','LFatalities4','LFatalities5',
'LConfirmedCasesMA1','LConfirmedCasesMA2','LConfirmedCasesMA3','LConfirmedCasesMA4','LConfirmedCasesMA5',
'LFatalitiesMA1','LFatalitiesMA2','LFatalitiesMA3','LFatalitiesMA4','LFatalitiesMA5']],train.LDConfirmedCases,categorical_feature=['CR','PS'])
 lgbm_f.fit(train.loc[:,['LDConfirmedCases1','LDConfirmedCases2','LDConfirmedCases3','LDConfirmedCases4','LDConfirmedCases5',
'LDFatalities1','LDFatalities2','LDFatalities3','LDFatalities4','LDFatalities5','days_since_confirmed','CR','PS',
'LDConfirmedCasesMA1','LDConfirmedCasesMA2','LDConfirmedCasesMA3','LDConfirmedCasesMA4','LDConfirmedCasesMA5',
'LDFatalitiesMA1','LDFatalitiesMA2','LDFatalitiesMA3','LDFatalitiesMA4','LDFatalitiesMA5',
'LConfirmedCases1','LConfirmedCases2','LConfirmedCases3','LConfirmedCases4','LConfirmedCases5',
'LFatalities1','LFatalities2','LFatalities3','LFatalities4','LFatalities5',
'LConfirmedCasesMA1','LConfirmedCasesMA2','LConfirmedCasesMA3','LConfirmedCasesMA4','LConfirmedCasesMA5',
'LFatalitiesMA1','LFatalitiesMA2','LFatalitiesMA3','LFatalitiesMA4','LFatalitiesMA5','LConfirmedCases','LDConfirmedCases']],train.LDFatalities,categorical_feature=['CR','PS'])
<feature_engineering>
blend_preds_df = pd.DataFrame()
blend_preds_df['id_seqpos'] = ensemble_final['id_seqpos']
blend_preds_df['reactivity'] = ensemble_final['reactivity']
blend_preds_df['deg_Mg_pH10'] = ensemble_final['deg_Mg_pH10']
blend_preds_df['deg_pH10'] = ensemble_final['deg_Mg_pH10']
blend_preds_df['deg_Mg_50C'] = ensemble_final['deg_Mg_50C']
blend_preds_df['deg_50C'] = ensemble_final['deg_Mg_50C']
blend_preds_df.head().T
<save_to_csv><EOS> 
blend_preds_df.to_csv('submission.csv', index=False )