Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<concatenate>
warnings.filterwarnings("ignore" )
x_train_cas = []
for i in range(len(x_train)) :
x = list(x_train[i])
 x.append(cases[i])
 x_train_cas.append(x)
 x_train_cas[0]<prepare_x_and_y>
seed = 42
x_train_cas = np.array(x_train_cas )<train_model>
dropout_model = 0.36
hidden_dim_first = 128
hidden_dim_second = 248
hidden_dim_third = 212
rf = XGBRegressor(n_estimators = 2500 , random_state = 0 , max_depth = 27)
 rf.fit(x_train_cas,fatalities )<concatenate>
commits_df = pd.DataFrame(columns = ['commit_num', 'dropout_model', 'hidden_dim_first', 'hidden_dim_second', 'hidden_dim_third', 'LB_score'] )
x_test_cas = []
for i in range(len(x_test)) :
x = list(x_test[i])
 x.append(cases_pred[i])
 x_test_cas.append(x)
 x_test_cas[0]<predict_on_test>
n=0
commits_df.loc[n,'commit_num'] = 0
commits_df.loc[n,'dropout_model'] = 0.4
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 128
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25883
fatalities_pred = rf.predict(x_test_cas)
 fatalities_pred<feature_engineering>
n=1
commits_df.loc[n,'commit_num'] = 3
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 256
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25855
fatalities_pred = np.around(fatalities_pred,decimals = 0)
 fatalities_pred<prepare_output>
n=2
commits_df.loc[n,'commit_num'] = 4
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 256
commits_df.loc[n,'hidden_dim_second'] = 256
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25887
submission['ConfirmedCases'] = cases_pred
submission['Fatalities'] = fatalities_pred<save_to_csv>
n=3
commits_df.loc[n,'commit_num'] = 5
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 384
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25880
submission.to_csv("submission.csv" , index = False )<import_modules>
n=4
commits_df.loc[n,'commit_num'] = 6
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 384
commits_df.loc[n,'hidden_dim_second'] = 128
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25989
from pandas_profiling import ProfileReport<load_from_csv>
n=5
commits_df.loc[n,'commit_num'] = 7
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 192
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25877
xtrain = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 xtest = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 xsubmission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )<create_dataframe>
n=6
commits_df.loc[n,'commit_num'] = 8
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 224
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25868
train_profile = ProfileReport(xtrain, title='Pandas Profiling Report', html={'style':{'full_width':True}})
 train_profile<prepare_x_and_y>
n=7
commits_df.loc[n,'commit_num'] = 9
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 288
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25881
xtrain.rename(columns={'Country_Region':'Country'}, inplace=True)
 xtest.rename(columns={'Country_Region':'Country'}, inplace=True)
 xtrain.rename(columns={'Province_State':'State'}, inplace=True)
 xtest.rename(columns={'Province_State':'State'}, inplace=True)
 xtrain['Date'] = pd.to_datetime(xtrain['Date'], infer_datetime_format=True)
 xtest['Date'] = pd.to_datetime(xtest['Date'], infer_datetime_format=True)
 xtrain.info()
xtest.info()
y1_xTrain = xtrain.iloc[:, -2]
y1_xTrain.head()
y2_xTrain = xtrain.iloc[:, -1]
y2_xTrain.head()
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state<data_type_conversions>
n=8
commits_df.loc[n,'commit_num'] = 10
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25850
X_xTrain = xtrain.copy()
X_xTrain['State'].fillna(EMPTY_VAL, inplace=True)
 X_xTrain['State'] = X_xTrain.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_xTrain.loc[:, 'Date'] = X_xTrain.Date.dt.strftime("%m%d")
 X_xTrain["Date"] = X_xTrain["Date"].astype(int)
 X_xTrain.head()
X_xTest = xtest.copy()
X_xTest['State'].fillna(EMPTY_VAL, inplace=True)
 X_xTest['State'] = X_xTest.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_xTest.loc[:, 'Date'] = X_xTest.Date.dt.strftime("%m%d")
 X_xTest["Date"] = X_xTest["Date"].astype(int)
 X_xTest.head()<categorify>
n=9
commits_df.loc[n,'commit_num'] = 11
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 240
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25868
le = preprocessing.LabelEncoder()
X_xTrain.Country = le.fit_transform(X_xTrain.Country)
 X_xTrain['State'] = le.fit_transform(X_xTrain['State'])
 X_xTrain.head()
X_xTest.Country = le.fit_transform(X_xTest.Country)
 X_xTest['State'] = le.fit_transform(X_xTest['State'])
 X_xTest.head()
xtrain.head()
xtrain.loc[xtrain.Country == 'Afghanistan', :]
xtest.tail()<categorify>
n=10
commits_df.loc[n,'commit_num'] = 12
commits_df.loc[n,'dropout_model'] = 0.35
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25884
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
countries = X_xTrain.Country.unique() 
<prepare_x_and_y>
n=11
commits_df.loc[n,'commit_num'] = 13
commits_df.loc[n,'dropout_model'] = 0.3
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25879
xout = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_xTrain.loc[X_xTrain.Country == country, :].State.unique()
for state in states:
X_xTrain_CS = X_xTrain.loc[(X_xTrain.Country == country)&(X_xTrain.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_xTrain_CS = X_xTrain_CS.loc[:, 'ConfirmedCases']
y2_xTrain_CS = X_xTrain_CS.loc[:, 'Fatalities']
X_xTrain_CS = X_xTrain_CS.loc[:, ['State', 'Country', 'Date']]
X_xTrain_CS.Country = le.fit_transform(X_xTrain_CS.Country)
 X_xTrain_CS['State'] = le.fit_transform(X_xTrain_CS['State'])
 X_xTest_CS = X_xTest.loc[(X_xTest.Country == country)&(X_xTest.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_xTest_CS_Id = X_xTest_CS.loc[:, 'ForecastId']
X_xTest_CS = X_xTest_CS.loc[:, ['State', 'Country', 'Date']]
X_xTest_CS.Country = le.fit_transform(X_xTest_CS.Country)
 X_xTest_CS['State'] = le.fit_transform(X_xTest_CS['State'])
 xmodel1 = XGBRegressor(n_estimators=1000)
 xmodel1.fit(X_xTrain_CS, y1_xTrain_CS)
 y1_xpred = xmodel1.predict(X_xTest_CS)
 xmodel2 = XGBRegressor(n_estimators=1000)
 xmodel2.fit(X_xTrain_CS, y2_xTrain_CS)
 y2_xpred = xmodel2.predict(X_xTest_CS)
 xdata = pd.DataFrame({'ForecastId': X_xTest_CS_Id, 'ConfirmedCases': y1_xpred, 'Fatalities': y2_xpred})
 xout = pd.concat([xout, xdata], axis=0)
<save_to_csv>
n=12
commits_df.loc[n,'commit_num'] = 15
commits_df.loc[n,'dropout_model'] = 0.45
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25920
xout.ForecastId = xout.ForecastId.astype('int')
 xout.tail()
xout.to_csv('submission.csv', index=False )<set_options>
n=13
commits_df.loc[n,'commit_num'] = 17
commits_df.loc[n,'dropout_model'] = 0.37
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25885
%matplotlib inline
%config InlineBackend.figure_format = 'retina' 
<load_from_csv>
n=14
commits_df.loc[n,'commit_num'] = 18
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 248
commits_df.loc[n,'LB_score'] = 0.25841
def get_cpmp_sub(save_oof=False, save_public_test=False):
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 train['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 train['day'] = train.Date.dt.dayofyear
train['geo'] = ['_'.join(x)for x in zip(train['Country_Region'], train['Province_State'])]
train
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test['Province_State'].fillna('', inplace=True)
 test['Date'] = pd.to_datetime(test['Date'])
 test['day'] = test.Date.dt.dayofyear
test['geo'] = ['_'.join(x)for x in zip(test['Country_Region'], test['Province_State'])]
test
day_min = train['day'].min()
train['day'] -= day_min
test['day'] -= day_min
min_test_val_day = test.day.min()
max_test_val_day = train.day.max()
max_test_day = test.day.max()
num_days = max_test_day + 1
min_test_val_day, max_test_val_day, num_days
train['ForecastId'] = -1
test['Id'] = -1
test['ConfirmedCases'] = 0
test['Fatalities'] = 0
debug = False
data = pd.concat([train,
test[test.day > max_test_val_day][train.columns]
] ).reset_index(drop=True)
 if debug:
data = data[data['geo'] >= 'France_'].reset_index(drop=True)
 gc.collect()
dates = data[data['geo'] == 'France_'].Date.values
if 0:
gr = data.groupby('geo')
 data['ConfirmedCases'] = gr.ConfirmedCases.transform('cummax')
 data['Fatalities'] = gr.Fatalities.transform('cummax')
 geo_data = data.pivot(index='geo', columns='day', values='ForecastId')
 num_geo = geo_data.shape[0]
geo_data
geo_id = {}
for i,g in enumerate(geo_data.index):
geo_id[g] = i
ConfirmedCases = data.pivot(index='geo', columns='day', values='ConfirmedCases')
 Fatalities = data.pivot(index='geo', columns='day', values='Fatalities')
 if debug:
cases = ConfirmedCases.values
deaths = Fatalities.values
else:
cases = np.log1p(ConfirmedCases.values)
 deaths = np.log1p(Fatalities.values)
 def get_dataset(start_pred, num_train, lag_period):
days = np.arange(start_pred - num_train + 1, start_pred + 1)
 lag_cases = np.vstack([cases[:, d - lag_period : d] for d in days])
 lag_deaths = np.vstack([deaths[:, d - lag_period : d] for d in days])
 target_cases = np.vstack([cases[:, d : d + 1] for d in days])
 target_deaths = np.vstack([deaths[:, d : d + 1] for d in days])
 geo_ids = np.vstack([geo_ids_base for d in days])
 country_ids = np.vstack([country_ids_base for d in days])
 return lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days
def update_valid_dataset(data, pred_death, pred_case):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
day = days[-1] + 1
new_lag_cases = np.hstack([lag_cases[:, 1:], pred_case])
 new_lag_deaths = np.hstack([lag_deaths[:, 1:], pred_death])
 new_target_cases = cases[:, day:day+1]
new_target_deaths = deaths[:, day:day+1]
new_geo_ids = geo_ids
new_country_ids = country_ids
new_days = 1 + days
return new_lag_cases, new_lag_deaths, new_target_cases, new_target_deaths, new_geo_ids, new_country_ids, new_days
def fit_eval(lr_death, lr_case, data, start_lag_death, end_lag_death, num_lag_case, fit, score):
lag_cases, lag_deaths, target_cases, target_deaths, geo_ids, country_ids, days = data
X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], country_ids])
 X_death = np.hstack([lag_deaths[:, -num_lag_case:], country_ids])
 X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death], lag_deaths[:, -num_lag_case:], country_ids])
 y_death = target_deaths
y_death_prev = lag_deaths[:, -1:]
if fit:
if 0:
keep =(y_death > 0 ).ravel()
X_death = X_death[keep]
y_death = y_death[keep]
y_death_prev = y_death_prev[keep]
lr_death.fit(X_death, y_death)
 y_pred_death = lr_death.predict(X_death)
 y_pred_death = np.maximum(y_pred_death, y_death_prev)
 X_case = np.hstack([lag_cases[:, -num_lag_case:], geo_ids])
 X_case = lag_cases[:, -num_lag_case:]
y_case = target_cases
y_case_prev = lag_cases[:, -1:]
if fit:
lr_case.fit(X_case, y_case)
 y_pred_case = lr_case.predict(X_case)
 y_pred_case = np.maximum(y_pred_case, y_case_prev)
 if score:
death_score = val_score(y_death, y_pred_death)
 case_score = val_score(y_case, y_pred_case)
 else:
death_score = 0
case_score = 0
return death_score, case_score, y_pred_death, y_pred_case
def train_model(train, valid, start_lag_death, end_lag_death, num_lag_case, num_val, score=True):
alpha = 3
lr_death = Ridge(alpha=alpha, fit_intercept=False)
 lr_case = Ridge(alpha=alpha, fit_intercept=True)
(train_death_score, train_case_score, train_pred_death, train_pred_case, 
)= fit_eval(lr_death, lr_case, train, start_lag_death, end_lag_death, num_lag_case, fit=True, score=score)
 death_scores = []
case_scores = []
death_pred = []
case_pred = []
for i in range(num_val): 
(valid_death_score, valid_case_score, valid_pred_death, valid_pred_case, 
)= fit_eval(lr_death, lr_case, valid, start_lag_death, end_lag_death, num_lag_case, fit=False, score=score)
 death_scores.append(valid_death_score)
 case_scores.append(valid_case_score)
 death_pred.append(valid_pred_death)
 case_pred.append(valid_pred_case)
 if 0:
print('val death: %0.3f' % valid_death_score,
'val case: %0.3f' % valid_case_score,
'val : %0.3f' % np.mean([valid_death_score, valid_case_score]),
flush=True)
 valid = update_valid_dataset(valid, valid_pred_death, valid_pred_case)
 if score:
death_scores = np.sqrt(np.mean([s**2 for s in death_scores]))
case_scores = np.sqrt(np.mean([s**2 for s in case_scores]))
if 0:
print('train death: %0.3f' % train_death_score,
'train case: %0.3f' % train_case_score,
'val death: %0.3f' % death_scores,
'val case: %0.3f' % case_scores,
'val : %0.3f' %(( death_scores + case_scores)/ 2),
flush=True)
 else:
print('%0.4f' % case_scores,
', %0.4f' % death_scores,
'= %0.4f' %(( death_scores + case_scores)/ 2),
flush=True)
 death_pred = np.hstack(death_pred)
 case_pred = np.hstack(case_pred)
 return death_scores, case_scores, death_pred, case_pred
countries = [g.split('_')[0] for g in geo_data.index]
countries = pd.factorize(countries)[0]
country_ids_base = countries.reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 country_ids_base = 0.2 * ohe.fit_transform(country_ids_base)
 country_ids_base.shape
geo_ids_base = np.arange(num_geo ).reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 geo_ids_base = 0.1 * ohe.fit_transform(geo_ids_base)
 geo_ids_base.shape
def val_score(true, pred):
pred = np.log1p(np.round(np.expm1(pred)- 0.2))
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
def val_score(true, pred):
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
start_lag_death, end_lag_death = 14, 6,
num_train = 5
num_lag_case = 14
lag_period = max(start_lag_death, num_lag_case)
 def get_oof(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[start_val], start_val, num_val)
 train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = train[['Date', 'Id', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[sub.day >= start_val]
sub = sub[['Id', 'ConfirmedCases', 'Fatalities']].copy()
return sub
if save_oof:
for start_val_delta, date in zip(range(3, -8, -3),
['2020-03-22', '2020-03-19', '2020-03-16', '2020-03-13']):
print(date, end=' ')
 oof = get_oof(start_val_delta)
 oof.to_csv('.. /submissions/cpmp-%s.csv' % date, index=None)
 def get_sub(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = 14
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data,
start_lag_death, end_lag_death, num_lag_case, num_val)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub.fillna(0)
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
return sub
return sub
known_test = train[['geo', 'day', 'ConfirmedCases', 'Fatalities']
].merge(test[['geo', 'day', 'ForecastId']], how='left', on=['geo', 'day'])
 known_test = known_test[['ForecastId', 'ConfirmedCases', 'Fatalities']][known_test.ForecastId.notnull() ].copy()
known_test
unknow_test = test[test.day > max_test_val_day]
unknow_test
def get_final_sub() :
start_val = max_test_val_day + 1
last_train = start_val - 1
num_val = max_test_day - start_val + 1
print(dates[last_train], start_val, num_val)
 num_lag_case = num_val + 3
train_data = get_dataset(last_train, num_train, lag_period)
 valid_data = get_dataset(start_val, 1, lag_period)
(_, _, val_death_preds, val_case_preds 
)= train_model(train_data, valid_data, start_lag_death, end_lag_death, num_lag_case, num_val, score=False)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
print(unknow_test.shape, pred_deaths.shape, pred_cases.shape)
 sub = unknow_test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
sub = pd.concat([known_test, sub])
 return sub
if save_public_test:
sub = get_sub()
else:
sub = get_final_sub()
return sub<load_from_csv>
n=15
commits_df.loc[n,'commit_num'] = 19
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 384
commits_df.loc[n,'LB_score'] = 0.25956
def get_nn_sub() :
df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 sub_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 coo_df = pd.read_csv(".. /input/countrydatafile/coordinates.csv" ).rename(columns={"Country/Region": "Country_Region"})
 coo_df = coo_df.groupby("Country_Region")[["Lat", "Long"]].mean().reset_index()
coo_df = coo_df[coo_df["Country_Region"].notnull() ]
loc_group = ["Province_State", "Country_Region"]
def preprocess(df):
df["Date"] = df["Date"].astype("datetime64[ms]")
 df["days"] =(df["Date"] - pd.to_datetime("2020-01-01")).dt.days
df["weekend"] = df["Date"].dt.dayofweek//5
df = df.merge(coo_df, how="left", on="Country_Region")
 df["Lat"] =(df["Lat"] // 30 ).astype(np.float32 ).fillna(0)
 df["Long"] =(df["Long"] // 60 ).astype(np.float32 ).fillna(0)
 for col in loc_group:
df[col].fillna("none", inplace=True)
 return df
df = preprocess(df)
 sub_df = preprocess(sub_df)
 print(df.shape)
 TARGETS = ["ConfirmedCases", "Fatalities"]
for col in TARGETS:
df[col] = np.log1p(df[col])
 NUM_SHIFT = 5
features = ["Lat", "Long"]
for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
df["prev_{}_{}".format(col, s)] = df.groupby(loc_group)[col].shift(s)
 features.append("prev_{}_{}".format(col, s))
df = df[df["Date"] >= df["Date"].min() + timedelta(days=NUM_SHIFT)].copy()
TEST_FIRST = sub_df["Date"].min()
TEST_DAYS =(df["Date"].max() - TEST_FIRST ).days + 1
dev_df, test_df = df[df["Date"] < TEST_FIRST].copy() , df[df["Date"] >= TEST_FIRST].copy()
def nn_block(input_layer, size, dropout_rate, activation):
out_layer = KL.Dense(size, activation=None )(input_layer)
 out_layer = KL.Activation(activation )(out_layer)
 out_layer = KL.Dropout(dropout_rate )(out_layer)
 return out_layer
def get_model() :
inp = KL.Input(shape=(len(features),))
hidden_layer = nn_block(inp, 128, 0.0, "relu")
 hidden_layer = nn_block(hidden_layer, 64, 0.0, "relu")
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 48, 0.0, "relu")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, "relu")
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(len(TARGETS), activation="linear" )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
get_model().summary()
def get_input(df):
return [df[features]]
NUM_MODELS = 10
def train_models(df, save=False):
models = []
for i in range(NUM_MODELS):
model = get_model()
model.compile(loss="mean_squared_error", optimizer=Nadam(lr=1e-4))
hist = model.fit(get_input(df), df[TARGETS],
batch_size=1024, epochs=500, verbose=0, shuffle=True)
 if save:
model.save_weights("model{}.h5".format(i))
models.append(model)
 return models
models = train_models(dev_df)
 prev_targets = ['prev_ConfirmedCases_1', 'prev_Fatalities_1']
def predict_one(df, models):
pred = np.zeros(( df.shape[0], 2))
for model in models:
pred += model.predict(get_input(df)) /len(models)
 pred = np.maximum(pred, df[prev_targets].values)
 pred[:, 0] = np.log1p(np.expm1(pred[:, 0])+ 0.1)
 pred[:, 1] = np.log1p(np.expm1(pred[:, 1])+ 0.01)
 return np.clip(pred, None, 15)
 print([mean_squared_error(dev_df[TARGETS[i]], predict_one(dev_df, models)[:, i])for i in range(len(TARGETS)) ])
 def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
def evaluate(df):
error = 0
for col in TARGETS:
error += rmse(df[col].values, df["pred_{}".format(col)].values)
 return np.round(error/len(TARGETS), 5)
 def predict(test_df, first_day, num_days, models, val=False):
temp_df = test_df.loc[test_df["Date"] == first_day].copy()
y_pred = predict_one(temp_df, models)
 for i, col in enumerate(TARGETS):
test_df["pred_{}".format(col)] = 0
test_df.loc[test_df["Date"] == first_day, "pred_{}".format(col)] = y_pred[:, i]
print(first_day, np.isnan(y_pred ).sum() , y_pred.min() , y_pred.max())
 if val:
print(evaluate(test_df[test_df["Date"] == first_day]))
y_prevs = [None]*NUM_SHIFT
for i in range(1, NUM_SHIFT):
y_prevs[i] = temp_df[['prev_ConfirmedCases_{}'.format(i), 'prev_Fatalities_{}'.format(i)]].values
for d in range(1, num_days):
date = first_day + timedelta(days=d)
 print(date, np.isnan(y_pred ).sum() , y_pred.min() , y_pred.max())
 temp_df = test_df.loc[test_df["Date"] == date].copy()
temp_df[prev_targets] = y_pred
for i in range(2, NUM_SHIFT+1):
temp_df[['prev_ConfirmedCases_{}'.format(i), 'prev_Fatalities_{}'.format(i)]] = y_prevs[i-1]
y_pred, y_prevs = predict_one(temp_df, models), [None, y_pred] + y_prevs[1:-1]
for i, col in enumerate(TARGETS):
test_df.loc[test_df["Date"] == date, "pred_{}".format(col)] = y_pred[:, i]
if val:
print(evaluate(test_df[test_df["Date"] == date]))
return test_df
test_df = predict(test_df, TEST_FIRST, TEST_DAYS, models, val=True)
 print(evaluate(test_df))
for col in TARGETS:
test_df[col] = np.expm1(test_df[col])
 test_df["pred_{}".format(col)] = np.expm1(test_df["pred_{}".format(col)])
 models = train_models(df, save=True)
 sub_df_public = sub_df[sub_df["Date"] <= df["Date"].max() ].copy()
sub_df_private = sub_df[sub_df["Date"] > df["Date"].max() ].copy()
pred_cols = ["pred_{}".format(col)for col in TARGETS]
sub_df_public = sub_df_public.merge(test_df[["Date"] + loc_group + TARGETS], how="left", on=["Date"] + loc_group)
 SUB_FIRST = sub_df_private["Date"].min()
SUB_DAYS =(sub_df_private["Date"].max() - sub_df_private["Date"].min() ).days + 1
sub_df_private = df.append(sub_df_private, sort=False)
 for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
sub_df_private["prev_{}_{}".format(col, s)] = sub_df_private.groupby(loc_group)[col].shift(s)
 sub_df_private = sub_df_private[sub_df_private["Date"] >= SUB_FIRST].copy()
sub_df_private = predict(sub_df_private, SUB_FIRST, SUB_DAYS, models)
 for col in TARGETS:
sub_df_private[col] = np.expm1(sub_df_private["pred_{}".format(col)])
 sub_df = sub_df_public.append(sub_df_private, sort=False)
 sub_df["ForecastId"] = sub_df["ForecastId"].astype(np.int16)
 return sub_df[["ForecastId"] + TARGETS]<data_type_conversions>
n=16
commits_df.loc[n,'commit_num'] = 20
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 128
commits_df.loc[n,'hidden_dim_third'] = 248
commits_df.loc[n,'LB_score'] = 0.25891
sub1 = get_cpmp_sub()
sub1['ForecastId'] = sub1['ForecastId'].astype('int' )<sort_values>
n=17
commits_df.loc[n,'commit_num'] = 21
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 240
commits_df.loc[n,'LB_score'] = 0.25831
sub1.sort_values("ForecastId", inplace=True)
 sub2.sort_values("ForecastId", inplace=True )<compute_test_metric>
n=18
commits_df.loc[n,'commit_num'] = 22
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 216
commits_df.loc[n,'LB_score'] = 0.25827
TARGETS = ["ConfirmedCases", "Fatalities"]
[np.sqrt(mean_squared_error(np.log1p(sub1[t].values), np.log1p(sub2[t].values)))for t in TARGETS]<save_to_csv>
n=19
commits_df.loc[n,'commit_num'] = 23
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 184
commits_df.loc[n,'LB_score'] = 0.25898
sub_df = sub1.copy()
for t in TARGETS:
sub_df[t] = np.expm1(np.log1p(sub1[t].values)*0.5 + np.log1p(sub2[t].values)*0.5)
 sub_df.to_csv("submission.csv", index=False )<load_from_csv>
n=20
commits_df.loc[n,'commit_num'] = 24
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 208
commits_df.loc[n,'LB_score'] = 0.25868
df=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/train.csv")
 test=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/test.csv" )<feature_engineering>
n=21
commits_df.loc[n,'commit_num'] = 25
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 212
commits_df.loc[n,'LB_score'] = 0.25823
df["Province_State"].fillna("state", inplace = True)
 df["Country_Region"] = [country_name.replace("'","")for country_name in df["Country_Region"]]
print(df.shape,"
",df.head() )<split>
n=22
commits_df.loc[n,'commit_num'] = 26
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 210
commits_df.loc[n,'LB_score'] = 0.25844
X_y=shuffle(new_data)
 y_cases=X_y['target_infection']
y_fatal=X_y['target_fatal']
X=X_y.drop(['target_infection','target_fatal'],axis=1)
 X_train_cases, X_test_cases, y_train_cases, y_test_cases = train_test_split(X, y_cases, test_size=0.33)
 X_train_fatal, X_test_fatal, y_train_fatal, y_test_fatal = train_test_split(X, y_fatal, test_size=0.33)
 print("Shape of infection train dataset:",(X_train_cases.shape,y_train_cases.shape))
print("Shape of infection test dataset:",(X_test_cases.shape,y_test_cases.shape))
print("Shape of fatal train dataset:",(X_train_fatal.shape,y_train_fatal.shape))
print("Shape of fatal test dataset:",(X_test_fatal.shape,y_test_fatal.shape))<train_on_grid>
n=23
commits_df.loc[n,'commit_num'] = 27
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 64
commits_df.loc[n,'hidden_dim_second'] = 124
commits_df.loc[n,'hidden_dim_third'] = 108
commits_df.loc[n,'LB_score'] = 0.25963
reg_case=ElasticNet(random_state=42,l1_ratio=0.1,max_iter=2200)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf = RandomizedSearchCV(reg_case, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search=clf.fit(X_train_cases, y_train_cases)
 results = pd.DataFrame.from_dict(clf.cv_results_ )<train_model>
n=24
commits_df.loc[n,'commit_num'] = 28
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 248
commits_df.loc[n,'hidden_dim_third'] = 108
commits_df.loc[n,'LB_score'] = 0.25860
best_alpha=1000
best_itr=2400
final_reg_case=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=best_itr)
 final_reg_case.fit(X_train_cases,y_train_cases )<predict_on_test>
n=25
commits_df.loc[n,'commit_num'] = 29
commits_df.loc[n,'dropout_model'] = 0.36
commits_df.loc[n,'hidden_dim_first'] = 128
commits_df.loc[n,'hidden_dim_second'] = 244
commits_df.loc[n,'hidden_dim_third'] = 128
commits_df.loc[n,'LB_score'] = 0.25846
pred=final_reg_case.predict(X_test_cases)
 print("The RMSE value",(mean_squared_error(y_test_cases,pred)) **0.5 )<train_on_grid>
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df['best'] = 0
commits_df.loc[commits_df['LB_score'].idxmin() , 'best'] = 1
reg_fatal=ElasticNet(random_state=42,l1_ratio=0.1,max_iter=3500)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf = RandomizedSearchCV(reg_fatal, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search=clf.fit(X_train_fatal, y_train_fatal)
 results = pd.DataFrame.from_dict(clf.cv_results_ )<split>
commits_df.sort_values(by=['LB_score'] )
featured=pd.DataFrame(data)
 X_y_f=shuffle(featured)
 y_cases_f=X_y_f['target_infection']
y_fatal_f=X_y_f['target_fatal']
X_f=X_y_f.drop(['target_infection','target_fatal'],axis=1)
 X_train_cases_f, X_test_cases_f, y_train_cases_f, y_test_cases_f = train_test_split(X_f, y_cases_f, test_size=0.33)
 X_train_fatal_f, X_test_fatal_f, y_train_fatal_f, y_test_fatal_f = train_test_split(X_f, y_fatal_f, test_size=0.33)
 print("Shape of featurized infection train dataset:",(X_train_cases_f.shape,y_train_cases_f.shape))
print("Shape of featurized infection test dataset:",(X_test_cases_f.shape,y_test_cases_f.shape))
print("Shape of featurized fatal train dataset:",(X_train_fatal_f.shape,y_train_fatal_f.shape))
print("Shape of featurized fatal test dataset:",(X_test_fatal_f.shape,y_test_fatal_f.shape))<train_on_grid>
train = pd.read_json('/kaggle/input/stanford-covid-vaccine/train.json', lines=True)
 test = pd.read_json('/kaggle/input/stanford-covid-vaccine/test.json', lines=True)
 sample_sub = pd.read_csv("/kaggle/input/stanford-covid-vaccine/sample_submission.csv" )
reg_case_f=ElasticNet(random_state=42,l1_ratio=0.1,max_iter=2200)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf_f= RandomizedSearchCV(reg_case_f, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search_f=clf_f.fit(X_train_cases_f, y_train_cases_f)
 results_f = pd.DataFrame.from_dict(clf_f.cv_results_ )<train_model>
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
best_alpha=10000
best_itr=4200
final_reg_case_f=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=best_itr)
 final_reg_case_f.fit(X_train_cases_f,y_train_cases_f )<predict_on_test>
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
def get_pair_index_structure(structure):
structure = np.array([struc for struc in structure], dtype="<U4")
 open_index = np.where(structure == "(")[0]
closed_index = np.where(structure == ")")[0]
structure[open_index] = range(0, len(open_index))
structure[closed_index] = range(len(open_index)-1, -1, -1)
 structure[structure == "."] = -1
structure = structure.astype(int)
 pair_structure = np.array([-1]*len(structure))
for i in range(len(open_index)) :
start, end = np.where(structure == i)[0]
pair_structure[start] = end
pair_structure[end] = start
return pair_structure
pred_f=final_reg_case_f.predict(X_test_cases_f)
 print("RMSE is:",(mean_squared_error(y_test_cases_f,pred_f)) **0.5 )<train_on_grid>
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
return np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 train_inputs_all = preprocess_inputs(train)
 train_labels_all = np.array(train[target_cols].values.tolist() ).transpose(( 0, 2, 1))
reg_fatal_f=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=2200)
 params = [{'alpha': [10**-4,10**-3, 10**-2,10**-1, 10**0,10**1, 10**2,10**3,10**4]}]
clf_f= RandomizedSearchCV(reg_fatal_f, params, cv=4, scoring='neg_root_mean_squared_error',return_train_score=True)
 search_f=clf_f.fit(X_train_fatal_f, y_train_fatal_f)
 results_f = pd.DataFrame.from_dict(clf_f.cv_results_ )<train_model>
def MCRMSE(y_true, y_pred):
colwise_mse = tf.reduce_mean(tf.square(y_true - y_pred), axis=1)
 return tf.reduce_mean(tf.sqrt(colwise_mse), axis=1)
 def gru_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.GRU(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer = 'orthogonal'))
def lstm_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.LSTM(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer = 'orthogonal'))
def build_model(model_type=1, seq_len=107, pred_len=68, embed_dim=100,
dropout=dropout_model, hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second, hidden_dim_third = hidden_dim_third):
inputs = tf.keras.layers.Input(shape=(seq_len, 3))
embed = tf.keras.layers.Embedding(input_dim=len(token2int), output_dim=embed_dim )(inputs)
 reshaped = tf.reshape(
 embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = tf.keras.layers.SpatialDropout1D (.2 )(reshaped)
 if model_type == 0:
hidden = gru_layer(hidden_dim_first, dropout )(reshaped)
 hidden = gru_layer(hidden_dim_second, dropout )(hidden)
 hidden = gru_layer(hidden_dim_third, dropout )(hidden)
 elif model_type == 1:
hidden = lstm_layer(hidden_dim_first, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim_second, dropout )(hidden)
 hidden = lstm_layer(hidden_dim_third, dropout )(hidden)
 elif model_type == 2:
hidden = gru_layer(hidden_dim_first, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim_second, dropout )(hidden)
 hidden = lstm_layer(hidden_dim_third, dropout )(hidden)
 elif model_type == 3:
hidden = lstm_layer(hidden_dim_first, dropout )(reshaped)
 hidden = gru_layer(hidden_dim_second, dropout )(hidden)
 hidden = gru_layer(hidden_dim_third, dropout )(hidden)
 elif model_type == 4:
hidden = lstm_layer(hidden_dim_first, dropout )(reshaped)
 hidden = gru_layer(hidden_dim_second, dropout )(hidden)
 hidden = lstm_layer(hidden_dim_third, dropout )(hidden)
 truncated = hidden[:, :pred_len]
out = tf.keras.layers.Dense(5, activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 adam = tf.optimizers.Adam()
model.compile(optimizer=adam, loss=MCRMSE)
 return model
best_alpha=100
best_itr=2400
final_reg_fatal_f=ElasticNet(random_state=42,alpha=best_alpha,l1_ratio=0.1,max_iter=best_itr)
 final_reg_fatal_f.fit(X_train_fatal_f,y_train_fatal_f )<predict_on_test>
def train_and_predict(n_folds=5, model_name="model", model_type=0, epochs=90, debug=False,
dropout_model=dropout_model, hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second, hidden_dim_third = hidden_dim_third,
seed=seed):
print("Model:", model_name)
 ensemble_preds = pd.DataFrame(index=sample_sub.index, columns=target_cols ).fillna(0)
 kf = KFold(n_folds, shuffle=True, random_state=seed)
 skf = StratifiedKFold(n_folds, shuffle=True, random_state=seed)
 val_losses = []
historys = []
for i,(train_index, val_index)in enumerate(skf.split(train_inputs_all, train['SN_filter'])) :
print("Fold:", str(i+1))
model_train = build_model(model_type=model_type,
dropout=dropout_model,
hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second,
hidden_dim_third = hidden_dim_third)
 model_short = build_model(model_type=model_type, seq_len=107, pred_len=107,
dropout=dropout_model,
hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second,
hidden_dim_third = hidden_dim_third)
 model_long = build_model(model_type=model_type, seq_len=130, pred_len=130,
dropout=dropout_model,
hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second,
hidden_dim_third = hidden_dim_third)
 train_inputs, train_labels = train_inputs_all[train_index], train_labels_all[train_index]
val_inputs, val_labels = train_inputs_all[val_index], train_labels_all[val_index]
checkpoint = tf.keras.callbacks.ModelCheckpoint(f'{model_name}.h5')
 history = model_train.fit(
 train_inputs , train_labels,
validation_data=(val_inputs,val_labels),
batch_size=64,
epochs=epochs,
callbacks=[tf.keras.callbacks.ReduceLROnPlateau() , checkpoint],
verbose=2 if debug else 0 
)
 print(f"{model_name} Min training loss={min(history.history['loss'])}, min validation loss={min(history.history['val_loss'])}")
 val_losses.append(min(history.history['val_loss']))
historys.append(history)
 model_short.load_weights(f'{model_name}.h5')
 model_long.load_weights(f'{model_name}.h5')
 public_preds = model_short.predict(public_inputs)
 private_preds = model_long.predict(private_inputs)
 preds_model = []
for df, preds in [(public_df, public_preds),(private_df, private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_model.append(single_df)
 preds_model_df = pd.concat(preds_model)
 ensemble_preds[target_cols] += preds_model_df[target_cols].values / n_folds
if debug:
print("Intermediate ensemble result")
 print(ensemble_preds[target_cols].head())
 ensemble_preds["id_seqpos"] = preds_model_df["id_seqpos"].values
ensemble_preds = pd.merge(sample_sub["id_seqpos"], ensemble_preds, on="id_seqpos", how="left")
 print("Mean Validation loss:", str(np.mean(val_losses)))
 if debug:
fig, ax = plt.subplots(1, 1, figsize =(10, 5))
for i, history in enumerate(historys):
ax.plot(history.history['loss'])
 ax.plot(history.history['val_loss'])
 ax.set_title('model_'+str(i+1))
ax.set_ylabel('Loss')
 ax.set_xlabel('Epoch')
 plt.show()
return ensemble_preds
public_df = test.query("seq_length == 107" ).copy()
private_df = test.query("seq_length == 130" ).copy()
public_inputs = preprocess_inputs(public_df)
 private_inputs = preprocess_inputs(private_df)
 ensembles = []
for i in range(5):
model_name = "model_"+str(i+1)
 ensemble = train_and_predict(n_folds=5, model_name=model_name, model_type=i, epochs=100,
dropout_model=dropout_model, hidden_dim_first = hidden_dim_first,
hidden_dim_second = hidden_dim_second, hidden_dim_third = hidden_dim_third,
seed=seed)
 ensembles.append(ensemble )
pred_f=final_reg_fatal_f.predict(X_test_fatal_f)
 print("RMSE is:",(mean_squared_error(y_test_fatal_f,pred_f)) **0.5 )<feature_engineering>
ensemble_final = ensembles[0].copy()
ensemble_final[target_cols] = 0
for ensemble in ensembles:
ensemble_final[target_cols] += ensemble[target_cols].values / len(ensembles)
 ensemble_final
<categorify><EOS> 
ensemble_final.to_csv('ensemble_final.csv', index=False )