Kaggle competition: COVID19 Global Forecasting (Week 4)
class save_best_n(tf.keras.callbacks.Callback):
def __init__(self, fn, model):
self.fn = fn
self.model = model
def on_epoch_end(self, epoch, logs=None):
if(epoch>0):
score=logs.get("val_auc")
 else:
score=-1
if(score > best_score[fold_num].min()):
idx_min=np.argmin(best_score[fold_num])
 best_score[fold_num][idx_min]=score
best_epoch[fold_num][idx_min]=epoch+1
path_best_model=f'best_model_fold_{self.fn}_{idx_min}.hdf5'
self.model.save(SAVE_FOLDER/path_best_model)
<categorify>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import nltk
from sklearn.preprocessing import LabelBinarizer,LabelEncoder,StandardScaler,MinMaxScaler
from sklearn.linear_model import LogisticRegression,SGDClassifier,LinearRegression
from sklearn.naive_bayes import MultinomialNB
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import classification_report,confusion_matrix,accuracy_score
from sklearn.model_selection import train_test_split
import keras
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from keras.models import Sequential
from keras.layers import Dense,LSTM
import tensorflow as tf
def setup_input(image, label, data, label_name):
tab_data=[tf.cast(data[tfeat], dtype=tf.float32)for tfeat in tab_feats]
tabular=tf.stack(tab_data)
 return {'inp1': image, 'inp2': tabular}, label<data_type_conversions>
train_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 test_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 submission = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv" )
hair_images_tf=tf.convert_to_tensor(hair_images)
 scale=tf.cast(IMAGE_SIZE[0]/256, dtype=tf.int32 )<categorify>
train_df.isna().sum()
def hair_aug(data, label):
img=tf.identity(data['inp1'])
 n_hairs = tf.random.uniform(shape=[], maxval=tf.constant(n_max)+1,
dtype=tf.int32)
 im_height=tf.shape(img)[1]
im_width=tf.shape(img)[0]
if n_hairs == 0:
return data, label
for _ in tf.range(n_hairs):
i=tf.random.uniform(shape=[], maxval=tf.shape(hair_images_tf)[0],
dtype=tf.int32)
 fname=hair_images_tf[i]
bits = tf.io.read_file(fname)
 hair = tf.image.decode_jpeg(bits)
 new_width=scale*tf.shape(hair)[1]
new_height=scale*tf.shape(hair)[0]
hair = tf.image.resize(hair, [new_height, new_width])
 hair = tf.image.random_flip_left_right(hair)
 hair = tf.image.random_flip_up_down(hair)
 n_rot=tf.random.uniform(shape=[], maxval=4,
dtype=tf.int32)
 hair = tf.image.rot90(hair, k=n_rot)
 h_height=tf.shape(hair)[0]
h_width=tf.shape(hair)[1]
roi_h0 = tf.random.uniform(shape=[], maxval=im_height - h_height + 1,
dtype=tf.int32)
 roi_w0 = tf.random.uniform(shape=[], maxval=im_width - h_width + 1,
dtype=tf.int32)
 roi = img[roi_h0:(roi_h0 + h_height), roi_w0:(roi_w0 + h_width)]
hair2gray = tf.image.rgb_to_grayscale(hair[:, :, :3])
 mask=hair2gray>10
img_bg = tf.multiply(roi, tf.cast(tf.image.grayscale_to_rgb(~mask),
dtype=tf.float32))
hair_fg = tf.multiply(tf.cast(hair[:, :, :3], dtype=tf.int32),
tf.cast(tf.image.grayscale_to_rgb(mask),
dtype=tf.int32 
)
)
 dst = tf.add(img_bg, tf.cast(hair_fg, dtype=tf.float32)/255)
 paddings = tf.stack([
[roi_h0, im_height-(roi_h0 + h_height)],
[roi_w0, im_width-(roi_w0 + h_width)],
[0, 0]
])
 dst_padded=tf.pad(dst, paddings, "CONSTANT")
 mask_img=tf.pad(tf.ones_like(dst), paddings, "CONSTANT")
 mask_img=~tf.cast(mask_img, dtype=tf.bool)
 img_hole=tf.multiply(img, tf.cast(mask_img, dtype=tf.float32))
img=tf.add(img_hole, dst_padded)
 return {'inp1': img, 'inp2': data['inp2']}, label<categorify>
test_df.isna().sum()
def get_training_dataset(dataset):
dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.map(hair_aug, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(512)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset<define_variables>
train_df['Country_Region'] = train_df['Country_Region'] + ' ' + train_df['Province_State']
test_df['Country_Region'] = test_df['Country_Region'] + ' ' + test_df['Province_State']
del train_df['Province_State']
del test_df['Province_State']
def get_validation_dataset(dataset):
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset<set_options>
def split_date(date):
date = date.split('-')
 date[0] = int(date[0])
 if(date[1][0] == '0'):
date[1] = int(date[1][1])
 else:
date[1] = int(date[1])
 if(date[2][0] == '0'):
date[2] = int(date[2][1])
 else:
date[2] = int(date[2])
 return date
train_df.Date = train_df.Date.apply(split_date)
 test_df.Date = test_df.Date.apply(split_date )
np.set_printoptions(threshold=15, linewidth=80 )<categorify>
year = []
month = []
day = []
for i in train_df.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2] )
def batch_to_numpy_images_and_labels(databatch, ds='train'):
if ds=='train':
data, labels = databatch
numpy_images = data['inp1'].numpy()
numpy_labels = labels.numpy()
else:
data = databatch
numpy_images = data['inp1'].numpy()
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels<define_variables>
train_df['Year'] = year
train_df['Month'] = month
train_df['Day'] = day
del train_df['Date']
def display_batch_of_images(databatch, predictions=None, ds='train'):

 images, labels = batch_to_numpy_images_and_labels(databatch, ds=ds)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else CLASSES[label]
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_image(image, title, subplot,
not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()<train_model>
year = []
month = []
day = []
for i in test_df.Date:
year.append(i[0])
 month.append(i[1])
 day.append(i[2] )
training_dataset = training_dataset.map(setup_input, num_parallel_calls=AUTO)
 training_dataset = get_training_dataset(training_dataset)
 training_dataset = training_dataset.unbatch().batch(20)
 train_batch = iter(training_dataset )<define_variables>
test_df['Year'] = year
test_df['Month'] = month
test_df['Day'] = day
del test_df['Date']
del train_df['Id']
del test_df['ForecastId']
display_batch_of_images(next(train_batch))<set_options>
del train_df['Year']
del test_df['Year']
del training_dataset, train_batch
gc.collect()<categorify>
train_df['ConfirmedCases'] = train_df['ConfirmedCases'].apply(int)
 train_df['Fatalities'] = train_df['Fatalities'].apply(int )
validation_dataset = validation_dataset.map(setup_input, num_parallel_calls=AUTO)
 validation_dataset = get_validation_dataset(validation_dataset)
 validation_dataset = validation_dataset.unbatch().batch(20)
 validation_batch = iter(validation_dataset )<define_variables>
cases = train_df.ConfirmedCases
fatalities = train_df.Fatalities
del train_df['ConfirmedCases']
del train_df['Fatalities']
display_batch_of_images(next(validation_batch))<set_options>
lb = LabelEncoder()
train_df['Country_Region'] = lb.fit_transform(train_df['Country_Region'])
 test_df['Country_Region'] = lb.transform(test_df['Country_Region'] )
del validation_dataset, validation_batch
gc.collect()<categorify>
scaler = MinMaxScaler()
x_train = scaler.fit_transform(train_df.values)
 x_test = scaler.transform(test_df.values )
def setup_test_image(image, data):
tab_data=[tf.cast(data[tfeat], dtype=tf.float32)for tfeat in tab_feats]
tabular=tf.stack(tab_data)
 return {'inp1': image, 'inp2': tabular}<define_variables>
from xgboost import XGBRegressor
def get_test_dataset(dataset):
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset<define_variables>
rf = XGBRegressor(n_estimators = 2500 , random_state = 0 , max_depth = 27)
 rf.fit(x_train,cases )
test_dataset = load_dataset(TEST_FNAMES, labeled=False, ordered=True)
 test_dataset = test_dataset.map(setup_test_image, num_parallel_calls=AUTO)
 test_dataset = get_test_dataset(test_dataset)
 test_dataset = test_dataset.unbatch().batch(20)
 test_batch = iter(test_dataset )<define_variables>
cases_pred = rf.predict(x_test)
 cases_pred
display_batch_of_images(next(test_batch), ds='test' )<set_options>
cases_pred = np.around(cases_pred,decimals = 0)
 cases_pred
del test_dataset, test_batch
gc.collect()<find_best_params>
x_train_cas = []
for i in range(len(x_train)) :
x = list(x_train[i])
 x.append(cases[i])
 x_train_cas.append(x)
 x_train_cas[0]
%%time
debug=0
histories = []
best_epoch={fn: np.zeros(NBEST)for fn in range(1, NFOLDS+1)}
best_score={fn: np.zeros(NBEST)for fn in range(1, NFOLDS+1)}
for fold_num in range(1, NFOLDS+1):
tf.keras.backend.clear_session()
tf.tpu.experimental.initialize_tpu_system(tpu)
 print("="*50)
 print(f"Starting fold {fold_num} out of {NFOLDS}...")
 files_trn=TRAIN_FNAMES[f"fold_{fold_num}"]
files_val=VAL_FNAMES[f"fold_{fold_num}"]
if debug:
files_trn=files_trn[0:2]
files_val=files_val[0:2]
EPOCHS=3
train_dataset = load_dataset(files_trn)
 train_dataset = train_dataset.map(setup_input, num_parallel_calls=AUTO)
 val_dataset = load_dataset(files_val, ordered = True)
 val_dataset = val_dataset.map(setup_input, num_parallel_calls=AUTO)
 model = get_model()
STEPS_PER_EPOCH = count_data_items(files_trn)// BATCH_SIZE
print(f'STEPS_PER_EPOCH = {STEPS_PER_EPOCH}')
 lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 history = model.fit(get_training_dataset(train_dataset),
steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[lr_callback,
save_best_n(fold_num, model),
],
validation_data=get_validation_dataset(val_dataset),
class_weight=class_weights,
verbose=2, 
)
 idx_sorted=np.argsort(best_score[fold_num])
 best_score[fold_num]=np.array(best_score[fold_num])[idx_sorted]
best_epoch[fold_num]=np.array(best_epoch[fold_num])[idx_sorted]
print(f"
Fold {fold_num} is finished.The best epochs: {[int(best_epoch[fold_num][i])for i in range(len(best_epoch[fold_num])) ]}")
 print(f"The corresponding scores: {[round(best_score[fold_num][i], 5)for i in range(len(best_epoch[fold_num])) ]}")
 histories.append(history )<find_best_params>
x_train_cas = np.array(x_train_cas )
for fn in range(1, NFOLDS+1):
display_training_curves(fn, data=histories[fn-1].history )<categorify>
rf = XGBRegressor(n_estimators = 2500 , random_state = 0 , max_depth = 27)
 rf.fit(x_train_cas,fatalities )
def setup_test_image(image, data):
tab_data=[tf.cast(data[tfeat], dtype=tf.float32)for tfeat in tab_feats]
tabular=tf.stack(tab_data)
 return {'inp1': image, 'inp2': tabular}<choose_model_class>
x_test_cas = []
for i in range(len(x_test)) :
x = list(x_test[i])
 x.append(cases_pred[i])
 x_test_cas.append(x)
 x_test_cas[0]
def average_predictions(X, fn):
y_probas=[]
for idx in range(NBEST):
tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 gc.collect()
print(f"Predicting: fold {fn}, model {idx+1} out of {NBEST}...")
 with strategy.scope() :
path_best_model=f'best_model_fold_{fn}_{idx}.hdf5'
model=tf.keras.models.load_model(SAVE_FOLDER/path_best_model)
 y=model.predict(X)
 y = rankdata(y)/len(y)
 y_probas.append(y)
 y_probas=np.average(y_probas, axis=0)
 return y_probas<prepare_x_and_y>
fatalities_pred = rf.predict(x_test_cas)
 fatalities_pred
%%time
preds = pd.DataFrame({'image_name': np.zeros(len(test)) , 'target': np.zeros(len(test)) })
 test_ds = load_dataset(TEST_FNAMES, labeled=False, ordered=True)
 test_images_ds = test_ds.map(setup_test_image, num_parallel_calls=AUTO)
 test_images_ds = get_test_dataset(test_images_ds)
 test_ds = get_test_dataset(test_ds)
 test_ids_ds = test_ds.map(setup_test_name, num_parallel_calls=AUTO ).unbatch()
preds['image_name'] = next(iter(test_ids_ds.batch(N_TEST_IMGS)) ).numpy().astype('U')
 preds['target'] = np.average([average_predictions(test_images_ds, fn)for fn in range(1, NFOLDS+1)], axis = 0 )<merge>
fatalities_pred = np.around(fatalities_pred,decimals = 0)
 fatalities_pred
del sub['target']
sub = sub.merge(preds, on='image_name')
 sub.head()<count_missing_values>
submission['ConfirmedCases'] = cases_pred
submission['Fatalities'] = fatalities_pred
print(f"The lengths of the submission file and `test` are {len(sub)} and {len(test)}, respectively.")
 print(f"The number of NA's in the submission file is {sub.isna().sum().sum() }." )<save_to_csv>
submission.to_csv("submission.csv" , index = False )