Kaggle competition: COVID19 Global Forecasting (Week 1)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-1)<data_type_conversions>
import matplotlib.pylab as plt
import numpy as np
import pandas as pd
import seaborn as sns
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<define_variables>
data_dir = '/kaggle/input/covid19-global-forecasting-week-1'
pd.read_csv(f'{data_dir}/train.csv', parse_dates=['Date'] ).head()
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 data_loader = DataLoader(
 dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)<categorify>
COLUMN_NAME_MAP = {
'Id': 'id',
'ForecastId': 'id',
'Province/State': 'province',
'Country/Region': 'country',
'Lat': 'lat',
'Long': 'long',
'Date': 'date',
'ConfirmedCases': 'cases',
'Fatalities': 'deaths'
}
def get_data(fn):

 df =(
 pd 
.read_csv(f'{data_dir}/{fn}.csv', parse_dates=['Date'])
.rename(columns=COLUMN_NAME_MAP)
.set_index('id')
.replace(pd.np.nan, '')
)
 df['day'] = df.date.dt.dayofyear
df['day'] -= 22
if 'deaths' in df:
df = df.astype({'deaths': int, 'cases': int})
 return df
train_df = get_data('train')
 train_df.head()
class BaseWheatTTA:

 image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTARotate180(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 2,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 2,(2, 3))
def deaugment_boxes(self, boxes):
boxes[:, [0,1,2,3]] = self.image_size - boxes[:, [2,3,0,1]]
return boxes
class TTARotate270(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 3,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 3,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = boxes[:, [1,3]]
res_boxes[:, [1,3]] = self.image_size - boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<concatenate>
train_df = train_df.loc[~bad_rows]
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , TTARotate180() , TTARotate270() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<train_on_grid>
countries_train_df = train_df.groupby(['country', 'date', 'day'])['cases', 'deaths'].sum().reset_index()
def load_net_eval(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path, map_location=DEVICE)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval()
return net.cuda()
model = load_net_eval('pl.bin' )<categorify>
most_cases_df = countries_train_df.groupby('country')['cases', 'deaths'].max().sort_values('cases', ascending=False)
 most_cases_df.head(10 )
def make_tta_predictions(images, score_threshold=0.25):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = model(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions<categorify>
test_df = get_data('test')
 x_predict = np.sort(test_df.day.unique())
 x_all = np.arange(x_predict.max() + 1)
 print(f'There are {len(x_predict)} to predict.')
 print(f'Overlap of {train_df.day.max() - test_df.day.min() } days.')
 test_df.head()
results = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
class ConstantModel:

 def __init__(self):
pass
def fit(self, X, y, sample_weight=None):
self.value = y[0]
def predict(self, X):
return np.array([self.value] * X.shape[0])
 def __repr__(self):
return f'ConstanModel(value={self.value})'
def run_models(grp_df, feature, target, x_to_predict, model, **model_kwargs):
region_data = grp_df[[feature, target]].to_numpy()
x = region_data[:, 0].reshape(-1, 1)
 y = region_data[:, 1]
n_target_classes = np.unique(y ).shape[0]
model = model(**model_kwargs)if n_target_classes > 1 else ConstantModel()
model.fit(x, y)
 result = {
feature: x_predict,
target: model.predict(x_to_predict.reshape(-1, 1)) ,
f'{target}_model': model
}
return pd.DataFrame(result ).set_index(feature )
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<load_from_csv>
region_columns = ['country', 'province']
death_predictions_df =(
 train_df 
.groupby(region_columns)
.apply(run_models, feature='day', target='deaths', x_to_predict=x_predict, model=model, **model_kwargs)
.reset_index() 
)
 cases_predictions_df =(
 train_df 
.groupby(region_columns)
.apply(run_models, feature='day', target='cases', x_to_predict=x_predict, model=model, **model_kwargs)
.reset_index()
)
train = pd.read_csv('.. /input/covid19-global-forecasting-week-1/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-1/test.csv')
 submission = pd.read_csv('.. /input/covid19-global-forecasting-week-1/submission.csv')
<data_type_conversions>
merge_columns = ['country', 'province', 'day']
result_df =(
 test_df 
.reset_index() 
.merge(death_predictions_df, on=merge_columns)
.merge(cases_predictions_df, on=merge_columns)
)
 result_df
test['Date']=test.Date.astype('datetime64[ns]' )<data_type_conversions>
evaluation_df = train_df.merge(
 result_df,
on=merge_columns,
how='inner',
suffixes=('', '_predicted')
)[merge_columns + ['cases', 'cases_predicted', 'deaths', 'deaths_predicted']]
for model_type in('cases', 'deaths',):
formula = f'log({model_type}_predicted + 1)- log({model_type} + 1)'
formula = f'({formula})**2'
evaluation_df = evaluation_df.eval(f'{model_type}_rmsle = {formula}')
 score_columns = ['cases_rmsle', 'deaths_rmsle'] 
(evaluation_df[score_columns].mean() ** 0.5 ).mean()
train['key']=train['Province/State'].astype('str')+ " " + train['Country/Region'].astype('str')+ " " +train['Lat'].astype('str')+ " " +train['Long'].astype('str')
 test['key']=test['Province/State'].astype('str')+ " " + test['Country/Region'].astype('str')+ " " +test['Lat'].astype('str')+ " " +test['Long'].astype('str')
 train.describe()<groupby>
(
evaluation_df 
.groupby(region_columns)[['cases_rmsle', 'deaths_rmsle']] 
.sum() 
.sort_values('cases_rmsle', ascending=False)
 )
daily_analysis=train.groupby(['Date'] ).sum()<compute_test_metric>
submission_df =(
 result_df 
.copy() 
.astype({'deaths': int, 'cases': int})
.rename(columns={v: k for k, v in COLUMN_NAME_MAP.items() })
.filter(['ForecastId', 'ConfirmedCases', 'Fatalities'])
)
 expected_submission_df = pd.read_csv(f'{data_dir}/submission.csv')
 for col in('ConfirmedCases', 'Fatalities',):
expected_submission_df[col] = submission_df[col]
pd.testing.assert_frame_equal(submission_df, expected_submission_df)
 submission_df.to_csv('submission.csv', index=False )
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<feature_engineering>
for model_type in('cases', 'deaths',):
train_df[f'daily_{model_type}'] = train_df.groupby(region_columns)[model_type].diff().fillna(0 ).astype(int)
 train_df.query('country == "United Kingdom" and province == "United Kingdom"' ).plot(x='day', y=f'daily_{model_type}' )
train_lag_1=train.groupby(['key'] ).shift(periods=1)
 train_lag_2=train.groupby(['key'] ).shift(periods=2)
 train_lag_3=train.groupby(['key'] ).shift(periods=3)
 train['lag_1_ConfirmedCases']=train_lag_1['ConfirmedCases']
train['lag_1_Fatalities']=train_lag_1['Fatalities']
train['lag_2_ConfirmedCases']=train_lag_2['ConfirmedCases']
train['lag_2_Fatalities']=train_lag_2['Fatalities']
train['lag_3_ConfirmedCases']=train_lag_3['ConfirmedCases']
train['lag_3_Fatalities']=train_lag_3['Fatalities']
train<create_dataframe>
mpl.rcParams['figure.figsize'] =(8, 6)
 mpl.rcParams['axes.grid'] = False
orig_stdout = sys.stdout
Fatalities_all_result_final=pd.DataFrame()
ConfirmedCases_all_result_Final=pd.DataFrame()
for keys in train['key'].unique() :
chk=train[train['key']==keys]
chk.index=chk.Date
fcastperiod=0
fcastperiod1=35
actual=chk[:chk.shape[0]-fcastperiod]
ffcast=chk[chk.shape[0]-fcastperiod-1:]
ffcast
try:
Fatalities_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'Fatalities' ).reset_index()
except:
Fatalities_all_result_1=pd.DataFrame(pd.date_range(start=chk.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 Fatalities_all_result_1.columns=['index']
Fatalities_all_result_1['best_model']='naive'
Fatalities_all_result_1['best_pred']=0
Fatalities_all_result_1['key']=keys
Fatalities_all_result_final=Fatalities_all_result_final.append(Fatalities_all_result_1,sort=True)
 try:
ConfirmedCases_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'ConfirmedCases' ).reset_index()
except:
ConfirmedCases_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 ConfirmedCases_all_result_1.columns=['index']
ConfirmedCases_all_result_1['best_model']='naive'
ConfirmedCases_all_result_1['best_pred']=1
ConfirmedCases_all_result_1['key']=keys
ConfirmedCases_all_result_Final=ConfirmedCases_all_result_Final.append(ConfirmedCases_all_result_1,sort=True)
 print(' done for %s' % keys)
 sys.stdout = orig_stdout<feature_engineering>
def univariate_data(df, start_index, end_index, history_size, target_size):

 data = []
labels = []
dataset = df.to_numpy()
start_index = start_index + history_size
if end_index is None:
end_index = len(dataset)- target_size
if end_index <= start_index:
raise ValueError(f'End index {end_index} not greater than start index {start_index}.')
 for i in range(start_index, end_index):
indices = range(i-history_size, i)
 data.append(dataset[indices].reshape(history_size, 1))
labels.append(dataset[i+target_size])
 return data, labels
def multivariate_data(dataset, target, start_index, end_index, history_size, target_size, step=1):
data = []
labels = []
dataset = dataset.to_numpy()
target = target.to_numpy()
start_index = start_index + history_size
if end_index is None:
end_index = len(dataset)- target_size
if end_index <= start_index:
raise ValueError(f'End index {end_index} not greater than start index {start_index}.')
 for i in range(start_index, end_index):
indices = range(i-history_size, i, step)
 data.append(dataset[indices].reshape(history_size, -1))
if step > 1:
labels.append(target[i:i+target_size])
 else:
labels.append(target[i+target_size])
 return data, labels
ConfirmedCases_all_result_Final.rename(columns={'index':'Date'},inplace=True)
 Fatalities_all_result_final.rename(columns={'index':'Date'},inplace=True)
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['AdditiveDamped'] is np.nan , ConfirmedCases_all_result_Final['best_pred'] ,
ConfirmedCases_all_result_Final['AdditiveDamped'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['AdditiveDamped'] is np.nan , Fatalities_all_result_final['best_pred'] ,
Fatalities_all_result_final['AdditiveDamped'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 ,
Fatalities_all_result_final['best_pred'] )<data_type_conversions>
history = 5
forecast_jump = 3
features = 'day'
scale = True
train_sample_size = test_df['day'].min()
scaled_df = train_df.copy()
if scale:
scaled_df[features] =(scaled_df[features] - scaled_df[features].mean())/ scaled_df[features].std()
xs = []
ys = []
regions = []
for grp_name, grp_df in scaled_df.groupby(['country', 'province']):
grp_x, grp_y = univariate_data(grp_df[features], 0, train_sample_size, history, forecast_jump)
 xs += grp_x
ys += grp_y
regions += [grp_name] * len(grp_y)
 break
x_train_uni = np.array(xs)
 y_train_uni = np.array(ys)
 regions_train = np.array(regions )
test['Date']=test.Date.astype('datetime64[ns]' )<merge>
print('Single window of past history')
 print(x_train_uni[0])
 print('
Target temperature to predict')
 print(y_train_uni[0] )
eval1 = ConfirmedCases_all_result_Final[['key','Date','best_pred','best_pred_1']].merge(test, how='right', on=['key','Date'])
 eval1.rename(columns={'best_pred':'ConfirmedCases'},inplace=True)
 eval1['ConfirmedCases']=eval1['ConfirmedCases'].fillna(0)
 eval1<merge>
scaled_df = train_df.copy()
if scale:
scaled_df[features] =(scaled_df[features] - scaled_df[features].mean())/ scaled_df[features].std()
xs = []
ys = []
regions = []
for grp_name, grp_df in scaled_df.groupby(['country', 'province']):
grp_x, grp_y = multivariate_data(
 grp_df[features], grp_df[features], train_sample_size, None, history, forecast_jump 
)
 xs += grp_x
ys += grp_y
regions += [grp_name] * len(grp_y)
 x_val_uni = np.array(xs)
 y_val_uni = np.array(ys)
 regions_val = np.array(regions )
eval2 = Fatalities_all_result_final[['key','Date','best_pred','best_pred_1']].merge(test, how='right', on=['key','Date'])
 eval2.rename(columns={'best_pred':'Fatalities'},inplace=True)
 eval2['Fatalities']=eval2['Fatalities'].fillna(0)
 eval2<merge>
i = 0
print('Single window of past history')
 print(x_val_uni[i])
 print('
Target temperature to predict')
 print(y_val_uni[i] )
sub_prep = eval1[['ForecastId','ConfirmedCases','key']].merge(eval2[['ForecastId','Fatalities']], on=['ForecastId'], how='left')
 sub_prep<merge>
BATCH_SIZE = 256
BUFFER_SIZE = 10000
train_univariate = tf.data.Dataset.from_tensor_slices(( x_train_uni, y_train_uni))
train_univariate = train_univariate.cache().shuffle(BUFFER_SIZE ).batch(BATCH_SIZE ).repeat()
val_univariate = tf.data.Dataset.from_tensor_slices(( x_val_uni, y_val_uni))
val_univariate = val_univariate.batch(BATCH_SIZE ).repeat()
sub = sub_prep.merge(submission['ForecastId'], on=['ForecastId'], how='right')
 sub<sort_values>
simple_lstm_model = tf.keras.models.Sequential([
tf.keras.layers.LSTM(8, input_shape=x_train_uni.shape[-2:]),
tf.keras.layers.Dense(1)
 ])
 simple_lstm_model.compile(optimizer='adam', loss='mae' )
<save_to_csv><EOS> 
EVALUATION_INTERVAL = 200
EPOCHS = 10
single_step_history = simple_lstm_model.fit(train_univariate, epochs=EPOCHS,
steps_per_epoch=EVALUATION_INTERVAL,
validation_data=val_univariate, validation_steps=50);