Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<load_from_csv>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sys
import os
import time
from tqdm import tqdm
from tqdm.keras import TqdmCallback
from datetime import timedelta
from copy import deepcopy
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Input, LSTM, Dense, Activation, Dropout
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
train_set = pd.read_csv(path + 'train.csv')
 train_labels = train_set['has_cactus']
train_labels = to_categorical(train_labels )<choose_model_class>
train_path = '.. /input/covid19-global-forecasting-week-4/train.csv'
test_path = '.. /input/covid19-global-forecasting-week-4/test.csv'
sub_path = '.. /input/covid19-global-forecasting-week-4/submission.csv'
model = models.Sequential()
model.add(layers.Conv2D(16,(3, 3), activation='relu', input_shape=(32, 32, 3), padding='same', kernel_initializer='he_uniform'))
model.add(layers.Conv2D(32,(3, 3), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.Conv2D(32,(3, 3), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.BatchNormalization())
 model.add(layers.MaxPooling2D(( 2, 2)))
 model.add(layers.Conv2D(64,(3, 3), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.Conv2D(64,(3, 3), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.Conv2D(64,(3, 3), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.BatchNormalization())
 model.add(layers.MaxPooling2D(( 2, 2)))
 model.add(layers.Conv2D(512,(3, 3), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.Conv2D(512,(3, 3), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.Conv2D(512,(3, 3), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.BatchNormalization())
 model.add(layers.MaxPooling2D(( 2, 2)))
 model.add(layers.Conv2D(32,(2, 2), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.Conv2D(32,(2, 2), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.Conv2D(32,(2, 2), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.BatchNormalization())
 model.add(layers.Conv2D(64,(2, 2), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.Conv2D(64,(2, 2), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.Conv2D(64,(2, 2), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.BatchNormalization())
 model.add(layers.MaxPooling2D(( 2, 2)))
 model.add(layers.Conv2D(128,(2, 2), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.Conv2D(128,(2, 2), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.Conv2D(128,(2, 2), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.Conv2D(128,(2, 2), activation='relu', padding='same', kernel_initializer='he_uniform'))
model.add(layers.BatchNormalization())
 model.add(layers.MaxPooling2D(( 2, 2)))
 model.add(layers.Dense(128, kernel_regularizer=l2(5e-4)))
 model.add(layers.Dense(64, kernel_regularizer=l2(5e-4)))
 model.add(layers.Dense(64, kernel_regularizer=l2(5e-4)))
 model.add(layers.Dense(32, kernel_regularizer=l2(5e-4)))
 model.add(layers.Flatten())
 model.add(layers.Dense(256))
model.add(layers.Dense(256))
model.add(layers.Dense(32))
model.add(layers.Dense(2, activation='softmax'))<choose_model_class>
df = pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv')
 df.tail()
cbl = [
EarlyStopping(
 monitor='val_loss',
patience=10, 
),
ModelCheckpoint(
 filepath='best_model.h5',
monitor='val_loss',
save_best_only=True, 
),
ReduceLROnPlateau(
 monitor='val_loss',
factor=0.1,
patience=10 
)
 ]<choose_model_class>
def load_csv(path):
df = pd.read_csv(path)
 df.fillna('None', inplace=True)
 return df
model.compile(optimizer=Adam() ,
loss='binary_crossentropy',
metrics=['accuracy', ],
)<train_model>
def timesteps(data, steps):
results = []
for i in range(len(data)- steps):
results.append(data[i:i+steps+1].values.tolist())
 return np.array(results)
 def input_reshape(data, shape):
return data.reshape(shape)
 def slide1_window(data, value):
data = data.reshape(-1, 2 ).tolist()
new_data = data[1:]
new_data.append(value)
 return np.array(new_data)
 def fit_lstm(train, vali, lstm_input_shape=None, batch_size=1, epochs=1, verbose=1):
X_train, y_train = train[:, :-1], train[:, -1]
X_train = input_reshape(X_train, lstm_input_shape)
 vali_data =(input_reshape(vali[:, :-1], lstm_input_shape), vali[:, -1])
 model = Sequential()
model.add(LSTM(128, input_shape=(lstm_input_shape[1:])))
 model.add(Dropout(0.2))
model.add(Dense(lstm_input_shape[-1]))
early_stopping = EarlyStopping(monitor='val_loss', patience=20)
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', patience=4, verbose=0, factor=0.5)
 callbacks = [early_stopping, reduce_lr]
if verbose != -1:
callbacks.append(TqdmCallback(verbose=verbose))
model.compile(optimizer='adam', loss='mse')
 model.fit(
 X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=0,
validation_data=vali_data,
callbacks=callbacks 
)
 return model
%%time
history = model.fit(
 train_images,
train_labels,
validation_split=0.01,
epochs=200,
batch_size=32,
callbacks=cbl
)<predict_on_test>
def load_cases(df, feat):
cases = pd.DataFrame(df[feat].values, index=df['Date'])
 return cases
def scale_fit(data):
scaler = MinMaxScaler()
scaler.fit(scale_reshape(data))
return scaler
def scale_transform(scaler, data):
return scaler.transform(scale_reshape(data)).reshape(data.shape)
 def scale_reshape(data):
results = np.array(data ).reshape(-1, 2)
 return results
class diff_node:
def __init__(self, first=None, v=None, level=0):
self.first = first
self.v = v
self.level = level
self.prev = None
self.next = None
def difference(X, degree):
diff = X
if degree == 0:
root = diff_node(v=diff)
 else:
root = diff_node(first=diff.iloc[0].values.reshape(-1,2))
for i in range(degree):
diff = diff.diff() [1:].reset_index(drop=True)
 if i + 1 >= degree:
root.next = diff_node(v=diff, level=i+1)
 else:
root.next = diff_node(first=diff.iloc[0].values.reshape(-1,2), level=i+1)
 root.next.prev = root
root = root.next
return root
def inverse_difference(node):
results = node.v
if node.level <= 1:
results = np.where(results >= 0.0, results, 0.0)
 while node.prev is not None:
prev = node.prev
first = prev.first
tmp = np.cumsum(results, axis=0)+ first
results = np.concatenate(( first, tmp), axis=0)
 if prev.level <= 1:
results = np.where(results >= 0.0, results, 0.0)
 node = prev
return np.array(results )
predicted_data = model.predict(test_images)
 predicted_data<load_from_csv>
class covid19_forecaster:
def __init__(self, feat, Country, Province, random_state=0):
self.scalar = None
self.model = None
self.model_input_shape = None
self.history = None
self.last_timestep = None
self.train_date = None
self.difference_node = None
self.feat = feat
self.Country = Country
self.Province = Province
self.random_state = random_state
def fit(self, df, time_steps=10, diff_degrees=0, vali_size=0.2, batch_size=1, epochs=1, verbose=1 
):
train_stepped = self._prepare_data(df, time_steps, diff_degrees)
 train_stepped_scaled = scale_transform(self.scalar, train_stepped)
 train, vali = train_test_split(train_stepped_scaled, test_size=vali_size, random_state=self.random_state)
 self.model_input_shape = -1, time_steps, len(self.feat)
 self.model = fit_lstm(train, vali,self.model_input_shape, batch_size, epochs, verbose)
 ')
 self.last_timestep = train_stepped_scaled[-1, :]
def forecast(self, forecast_steps):
predictions_scaled = []
X_test, y_test = self.last_timestep[:-1], self.last_timestep[-1]
for i in range(forecast_steps):
X_test = slide1_window(X_test, y_test)
 y_test = self.model.predict(X_test.reshape(self.model_input_shape)) [0, :] * 1.03
predictions_scaled.append(y_test)
 predictions_scaled = np.array(predictions_scaled)
 predictions = self._prediction_postprocess(self.scalar, predictions_scaled, self.difference_node)
 start_date = pd.to_datetime(self.train_date[-1], format='%Y-%m-%d')
 pred_dates = self._generate_dates(start_date, forecast_steps)
 return pd.DataFrame(predictions, index=pred_dates, columns=self.feat)
 def _prepare_data(self, df, time_steps, diff_degrees):
cases = load_cases(df, self.feat)
 self.history = cases
self.difference_node = difference(cases, diff_degrees)
 difference_value = self.difference_node.v
self.scalar = scale_fit(difference_value)
 train_stepped = timesteps(difference_value, time_steps)
 self.train_date = cases.index[time_steps:]
return train_stepped
def _prediction_postprocess(self, scaler, predictions, node, append=True):
preds = scaler.inverse_transform(predictions.reshape(-1, len(self.feat)))
 preds = preds.reshape(-1, len(self.feat))
tmp_node = deepcopy(node)
 if append:
tmp_node.v = tmp_node.v.append(pd.DataFrame(preds), ignore_index=True)
 preds = inverse_difference(tmp_node)[-len(predictions):]
else:
tmp_node.v = preds
preds = inverse_difference(tmp_node)
 return np.array(preds)
 Province_State: %s'%(self.Country, self.Province))
def _generate_dates(self, start_date, length):
dates = []
for i in range(length):
new_date = start_date + timedelta(days=int(i + 1))
dates.append(new_date.strftime('%Y-%m-%d'))
return dates
def _indextodatetime(self, *datas):
results = []
for data in datas:
dates = data.index.to_series().apply(lambda x: pd.to_datetime(x, format='%Y-%m-%d')).values
values = data.values
new_data = pd.Series(values, index=dates)
 results.append(new_data)
 return tuple(results )
test_data = pd.read_csv(path + 'sample_submission.csv')
 test_data['has_cactus'] = predicted_data<save_to_csv>
def forecast(train_path, test_path):
df_train = load_csv(train_path)
 df_test = load_csv(test_path)
 cp = []
for i, row in df_test.iterrows() :
v = row['Country_Region'], row['Province_State']
if v not in cp:
cp.append(v)
 feats = ['ConfirmedCases', 'Fatalities']
train_start = 40
train_end = 84
time_steps = 10
diff_degrees = 1
vali_size = 0.15
batch_size = 8
epochs = 200
verbose = -1
forecast_steps = 30
predictions = []
for Country, Province in tqdm(cp):
df = df_train[(df_train['Country_Region']==Country)&(df_train['Province_State']==Province)]
train = df[train_start:train_end]
random_state = np.random.randint(0, 20, 1)[0]
forecaster = covid19_forecaster(feats, Country, Province, random_state=random_state)
 forecaster.fit(train, time_steps=time_steps, diff_degrees=diff_degrees, vali_size=vali_size,
batch_size=batch_size, epochs=epochs, verbose=verbose)
 preds = forecaster.forecast(forecast_steps)
 preds = preds.values.tolist()
for v in train[-13:][feats].values.tolist() :
predictions.append(v)
 for p in preds:
predictions.append(p)
 return predictions
test_data.to_csv('sample_submission.csv', index=False )<set_options>
predictions = forecast(train_path, test_path )
%matplotlib inline
random.seed(0)
<prepare_x_and_y>
df_sub = pd.read_csv(sub_path)
 feats = ['ConfirmedCases', 'Fatalities']
df_sub[feats] = predictions
df_sub.to_csv('submission.csv', index=False )