Kaggle competition: COVID19 Global Forecasting (Week 3)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-week-3<filter>
path = '.. /input/covid19-global-forecasting-week-3/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'submission.csv')
 train['Date'] = train['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 test['Date'] = test['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 train['days'] =(train['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
test['days'] =(test['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
train.loc[train['Province_State'].isnull() , 'Province_State'] = 'N/A'
test.loc[test['Province_State'].isnull() , 'Province_State'] = 'N/A'
train['Area'] = train['Country_Region'] + '_' + train['Province_State']
test['Area'] = test['Country_Region'] + '_' + test['Province_State']
print(train['Date'].max())
 print(test['Date'].min())
 print(train['days'].max())
 N_AREAS = train['Area'].nunique()
AREAS = np.sort(train['Area'].unique())
 TRAIN_N = 77
print(train[train['days'] < TRAIN_N]['Date'].max())
 train.head()
TEST_FILENAMES=tf.io.gfile.glob(test_tf_records_path + '/*test.rec' )<normalization>
def eval1(y, p):
val_len = y.shape[1] - TRAIN_N
return np.sqrt(mean_squared_error(y[:, TRAIN_N:TRAIN_N+val_len].flatten() , p[:, TRAIN_N:TRAIN_N+val_len].flatten()))
def run_c(params, X, test_size=50):
gr_base = []
gr_base_factor = []
x_min = np.ma.MaskedArray(X, X<1)
 x_min = x_min.argmin(axis=1)
 for i in range(X.shape[0]):
temp = X[i,:]
threshold = np.log(1+params['min cases for growth rate'])
 num_days = params['last N days']
if(temp > threshold ).sum() > num_days:
gr_base.append(np.clip(np.diff(temp[temp > threshold])[-num_days:].mean() , 0, params['growth rate max']))
gr_base_factor.append(np.clip(np.diff(np.diff(temp[temp > threshold])) [-num_days:].mean() , -0.2, params["growth rate factor max"]))
else:
gr_base.append(params['growth rate default'])
 gr_base_factor.append(params['growth rate factor'])
 gr_base = np.array(gr_base)
 gr_base_factor = np.array(gr_base_factor)
 preds = X.copy()
for i in range(test_size):
delta = np.clip(preds[:, -1], np.log(2), None)+ gr_base *(1 + params['growth rate factor']*(1 + params['growth rate factor factor'])**(i)) **(i)
 preds = np.hstack(( preds, delta.reshape(-1,1)))
 return preds
params = {
"min cases for growth rate": 30,
"last N days": 6,
"growth rate default": 0.3,
"growth rate max": 0.35,
"growth rate factor max": -0.01,
"growth rate factor": -0.05,
"growth rate factor factor": 0.005,
}
x = train_p_c[train_p_c.index=="China_Qinghai"]
x = train_p_c
preds_c_1 = run_c(params, np.log(1+x.values)[:,:TRAIN_N])
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [img_size,img_size, 3])
 return image
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 if label is None:
return image
else:
return image, label
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset<create_dataframe>
warnings.filterwarnings("ignore")
 def f(x, K, a, x0):
return K * x ** a * np.exp(-x/x0)
 def eval1(y, p):
val_len = y.shape[1] - TRAIN_N
return np.sqrt(mean_squared_error(y[:, TRAIN_N:TRAIN_N+val_len].flatten() , p[:, TRAIN_N:TRAIN_N+val_len].flatten()))
def run(params, X, X_change, test_size=50):
print(X)
 x_mins = np.ma.MaskedArray(X, X<10)
 x_mins = x_mins.argmin(axis=1)
 print(x_mins)
 popts = []
scores = []
for i in tqdm(range(X_change.shape[0])) :
if X[i,:].sum() == 0:
x_mins[i] = X.shape[1] - 1
best_score = 100
best_popt = [1,1,1]
x_min_max = X_change.shape[1]
x_min_max = np.minimum(x_mins[i] + 10, X_change.shape[1])
 early_stopping = 5
early_stopping_count = 0
for x_min in np.arange(x_mins[i], x_min_max):
x = np.arange(x_min, X_change.shape[1])- x_min
y = X_change[i, x_min:]
try:
popt, pcov = curve_fit(f, x,y, bounds=(0, [10, 5, np.max(x)]))
except:
popt = [1,1,1]
p = np.zeros(X_change.shape[1])
 p[x_min:] = f(x, *popt)
 p = np.cumsum(p, axis=0)
 score = np.sqrt(mean_squared_error(np.log1p(X[i,:]), np.log1p(p)))
 if score < best_score:
best_score = score
best_popt = popt
x_mins[i] = x_min
else:
early_stopping_count += 1
if early_stopping_count >= early_stopping:
continue
popts.append(best_popt)
 scores.append(best_score)
 preds = X_change.copy()
preds_new = np.zeros(( X.shape[0], test_size))
for i in range(X.shape[0]):
x = np.arange(X.shape[1], test_size+X.shape[1])- x_mins[i]
y = f(x, *popts[i])
 y = y[-test_size-X.shape[1]:]
preds_new[i,:] = y
preds = np.hstack(( preds, preds_new))
preds = np.cumsum(preds, axis=1)
 return preds, x_mins, scores
x1 = train_p_c
x = train_p_c.values[:,:TRAIN_N]
x_c = train_p_c_change.values[:,:TRAIN_N]
params = {}
preds_c_pl, x_mins, scores = run(params, x, x_c)
 preds_c_pl = np.log1p(preds_c_pl)
train_dataset=get_training_dataset() 
<load_from_csv>
preds_c = preds_c_1.copy()
idx = np.where(x_mins<100)
 preds_c[idx] = 0.8 * preds_c_1[idx] + 0.2 * preds_c_pl[idx]
for i in range(N_AREAS):
if 'China' in AREAS[i] and preds_c[i, TRAIN_N-1] < np.log(31):
preds_c[i, TRAIN_N:] = preds_c[i, TRAIN_N-1]
sub_df = pd.read_csv('.. /input/iwildcam-2020-fgvc7/sample_submission.csv')
 sub_df.head()<create_dataframe>
f_rate =(train_p_f / train_p_c ).fillna(0)
 X_c = np.log(1+train_p_c.values)[:,:TRAIN_N]
X_f = train_p_f.values[:,:TRAIN_N]
test_dataset=get_test_dataset() 
<choose_model_class>
def lin_w(sz):
res = np.linspace(0, 1, sz+1, endpoint=False)[1:]
return np.append(res, np.append([1], res[::-1]))
def run_f(params, X_c, X_f, X_f_r, test_size=50):
X_f_r = np.array(np.ma.mean(np.ma.masked_outside(X_f_r, 0.06, 0.4)[:,:], axis=1))
X_f_r = np.clip(X_f_r, params['fatality_rate_lower'], params['fatality_rate_upper'])
 X_c = np.clip(np.exp(X_c)-1, 0, None)
 preds = X_f.copy()
train_size = X_f.shape[1] - 1
for i in range(test_size):
t_lag = train_size+i-params['length']
t_wsize = 3
delta = np.average(np.diff(X_c, axis=1)[:, t_lag-t_wsize:t_lag+1+t_wsize], axis=1)
 delta = params['absolute growth'] + delta * X_f_r
preds = np.hstack(( preds, preds[:, -1].reshape(-1,1)+ delta.reshape(-1,1)))
 return preds
params = {
"length": 6,
"absolute growth": 0.02,
"fatality_rate_lower": 0.035,
"fatality_rate_upper": 0.40,
}
preds_f_1 = run_f(params, preds_c, X_f, f_rate.values[:,:TRAIN_N])
 preds_f_1 = np.log(1+preds_f_1)
 preds_f_1 = np.log1p(0.9*(np.exp(preds_f_1)-1))
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB7(
 input_shape=(img_size, img_size, 3),
weights='imagenet',
include_top=False 
),
L.GlobalAveragePooling2D() ,
L.Dense(216, activation='softmax')
 ])
 model.compile(
 optimizer='adam',
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model.summary() 
<choose_model_class>
preds_f = preds_f_1
early = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=1, mode='auto')
 best_checkpoint='model.h5'
checkpoint = ModelCheckpoint(
 best_checkpoint,
monitor='val_accuracy',
verbose=1,
save_weights_only=True,
mode='auto'
)<train_model>
if False:
val_len = train_p_c.values.shape[1] - TRAIN_N
for i in range(val_len):
d = i + TRAIN_N
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c.values[:, d]), preds_c[:, d]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f.values[:, d]), preds_f[:, d]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_c[:, TRAIN_N:TRAIN_N+val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_f[:, TRAIN_N:TRAIN_N+val_len].flatten()))
print(f"{(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]" )
%%time
STEPS_PER_EPOCH=(143736+3709)// BATCH_SIZE
history = model.fit(
 train_dataset,
epochs=EPOCHS,
verbose=2,
callbacks=[early,checkpoint],
steps_per_epoch=STEPS_PER_EPOCH,
)<set_options>
temp = pd.DataFrame(np.clip(np.exp(preds_c)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="ConfirmedCases")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 temp = pd.DataFrame(np.clip(np.exp(preds_f)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="Fatalities")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 test.head()
gc.collect()<prepare_output>
test.to_csv("submission.csv", index=False, columns=["ForecastId", "ConfirmedCases", "Fatalities"] )
<predict_on_test><EOS> 
for i, rec in test.groupby('Area' ).last().sort_values("ConfirmedCases", ascending=False ).iterrows() :
print(f"{rec['ConfirmedCases']:10.1f} {rec['Fatalities']:10.1f} {rec['Country_Region']}, {rec['Province_State']}")