Kaggle competition: COVID19 Global Forecasting (Week 4)
if colab:
OUT_FOLDER=SAVE_FOLDER
else:
OUT_FOLDER=Path('')
 sub.to_csv(OUT_FOLDER/'submission.csv', index=False )<train_model>
train_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 test_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 submission = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv" )
def setup_val_image(image, label, data, label_name):
tab_data=[tf.cast(data[tfeat], dtype=tf.float32)for tfeat in tab_feats]
tabular=tf.stack(tab_data)
 return {'inp1': image, 'inp2': tabular}<train_model>
train_df['Province_State'].fillna("",inplace = True)
 test_df['Province_State'].fillna("",inplace = True)
 train_df['Country_Region'] = train_df['Country_Region'] + ' ' + train_df['Province_State']
test_df['Country_Region'] = test_df['Country_Region'] + ' ' + test_df['Province_State']
del train_df['Province_State']
del test_df['Province_State']
def setup_val_label(image, label, data, label_name):
return label<define_variables>
def split_date(date):
date = date.split('-')
 date[0] = int(date[0])
 if(date[1][0] == '0'):
date[1] = int(date[1][1])
 else:
date[1] = int(date[1])
 if(date[2][0] == '0'):
date[2] = int(date[2][1])
 else:
date[2] = int(date[2])
 return date
train_df.Date = train_df.Date.apply(split_date)
 test_df.Date = test_df.Date.apply(split_date )
oof= pd.DataFrame({'image_name': train['image_name'].values})
 auc=[]
oof_all_folds=[]
for fold_num in range(1, NFOLDS+1):
tf.keras.backend.clear_session()
print("="*50)
 print(f"Starting fold {fold_num}...")
 print(f"The best epochs: {[int(best_epoch[fold_num][i])for i in range(len(best_epoch[fold_num])) ]}")
 print(f"The corresponding scores: {[round(best_score[fold_num][i], 5)for i in range(len(best_epoch[fold_num])) ]}")
 files_val=VAL_FNAMES[f"fold_{fold_num}"]
if debug:
files_val=files_val[0:2]
val_ds = load_dataset(files_val, ordered = True)
 val_images_ds = val_ds.map(setup_val_image,
num_parallel_calls=AUTO)
 val_images_ds = get_validation_dataset(val_images_ds)
 val_ds = get_validation_dataset(val_ds)
 val_label_ds = val_ds.map(setup_val_label,
num_parallel_calls=AUTO ).unbatch()
val_ids_ds = val_ds.map(setup_val_name,
num_parallel_calls=AUTO ).unbatch()
n_val_fold = count_data_items(files_val)
 print(f'The
oof_fold= pd.DataFrame()
oof_fold['image_name'] = next(iter(val_ids_ds.batch(n_val_fold)) ).numpy().astype('U')
 oof_fold['target'] = average_predictions(val_images_ds, fold_num)
 oof_all_folds.append(oof_fold)
 y_true = next(iter(val_label_ds.batch(n_val_fold)) ).numpy()
auc_fold=roc_auc_score(y_true, oof_fold['target'].values)
 auc.append(auc_fold)
 print(f"Fold {fold_num} is done! ROC AUC = {auc_fold:.5f}")
 oof=oof.merge(pd.concat(oof_all_folds),
on='image_name',
how='left' ).reset_index(drop=True)
 if debug:
oof=oof.fillna(0)
 auc=np.array(auc)
 auc_av=auc.mean()
auc_std=auc.std()
print(f"ROC AUC = {auc_av:.5f}, STDEV = {auc_std:.5f}(average across the folds)")
 auc_oof=roc_auc_score(train['target'].values, oof['target'].values)
 print(f"ROC AUC = {auc_oof:.5f}(out of folds)" )<count_missing_values>
year = []
month = []
day = []
for i in train_df.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2] )
print(f"The lengths of the oof data frame and `train` are {len(oof)} and {len(train)}, respectively.")
 print(f"The number of NA's in the oof data frame is {oof.isna().sum().sum() }." )<save_to_csv>
train_df['Year'] = year
train_df['Month'] = month
train_df['Day'] = day
del train_df['Date']
oof.to_csv(OUT_FOLDER/'oof.csv', index=False )<load_from_csv>
year = []
month = []
day = []
for i in test_df.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2] )
<load_from_csv>
test_df['Year'] = year
test_df['Month'] = month
test_df['Day'] = day
del test_df['Date']
del train_df['Id']
del test_df['ForecastId']
sub_v9=pd.read_csv('/kaggle/input/siim-benchmark-submission/submission_v9.csv')['target'].values
sub_v10=pd.read_csv('/kaggle/input/siim-benchmark-submission/submission_v10.csv')['target'].values
sub_v13=pd.read_csv('/kaggle/input/siim-benchmark-submission/submission_v10.csv')['target'].values
sub_v9=rankdata(sub_v9)/len(sub_v9)
 sub_v10=rankdata(sub_v10)/len(sub_v10)
 sub_v13=rankdata(sub_v13)/len(sub_v13)
 oof_v9=pd.read_csv('/kaggle/input/siim-benchmark-submission/oof_v9.csv')['target'].values
oof_v10=pd.read_csv('/kaggle/input/siim-benchmark-submission/oof_v10.csv')['target'].values
oof_v13=pd.read_csv('/kaggle/input/siim-benchmark-submission/oof_v13.csv')['target'].values
oof_v9=rankdata(oof_v9)/len(oof_v9)
 oof_v10=rankdata(oof_v10)/len(oof_v10)
 oof_v13=rankdata(oof_v13)/len(oof_v13)
 preds_blend=1/2*1/3*(sub_v9+
sub_v10+
sub['target'].values 
)+1/2*sub_v13
oof_blend=1/2*1/3*(oof_v9+
oof_v10+
oof['target'].values 
)+1/2*oof_v13
auc_blend=roc_auc_score(train['target'].values, oof_blend)
 print(f"ROC AUC score of the blend = {auc_blend:.5f}" )<save_to_csv>
rf = XGBRegressor(n_estimators = 1600 , random_state = 0 , max_depth = 15)
 rf.fit(x_train,cases)
 cases_pred = rf.predict(x_test)
 cases_pred = np.around(cases_pred,decimals = 0)
 x_train_cas = []
for i in range(len(x_train)) :
x = list(x_train[i])
 x.append(cases[i])
 x_train_cas.append(x)
 x_train_cas[0]
x_train_cas = np.array(x_train_cas)
 rf = XGBRegressor(n_estimators = 1600 , random_state = 0 , max_depth = 15)
 rf.fit(x_train_cas,fatalities)
 x_test_cas = []
for i in range(len(x_test)) :
x = list(x_test[i])
 x.append(cases_pred[i])
 x_test_cas.append(x)
 x_test_cas[0]
x_test_cas = np.array(x_test_cas)
 fatalities_pred = rf.predict(x_test_cas)
 fatalities_pred = np.around(fatalities_pred,decimals = 0)
 submission['ConfirmedCases'] = cases_pred
submission['Fatalities'] = fatalities_pred
submission.to_csv("submission.csv" , index = False )