Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<load_from_csv>
import os, traceback
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
from sklearn.linear_model import LinearRegression, Ridge
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import PolynomialFeatures, StandardScaler
from sklearn.svm import LinearSVR
from sklearn.utils import shuffle
df_train = pd.read_csv(".. /input/train_V2.csv")
 df_test = pd.read_csv(".. /input/test_V2.csv")
 print("Train data set :
",df_Train.head())
 print("Test data set :
", df_Test.head() )<feature_engineering>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
path = os.path.join(dirname, filename)
 if 'train' in filename:
train = pd.read_csv(path)
 elif 'test' in filename:
test = pd.read_csv(path)
 scaler = StandardScaler()
df_train.loc[df_train['winPlacePerc'].isna() ==True, 'winPlacePerc']= 0.5<feature_engineering>
train.rename(columns={'Country_Region':'Country',
'Province_State': 'State'},
inplace=True)
 test.rename(columns={'Country_Region':'Country',
'Province_State': 'State'},
inplace=True)
 train['Date'] = pd.to_datetime(train['Date'], infer_datetime_format=True)
 test['Date'] = pd.to_datetime(test['Date'], infer_datetime_format=True)
 submission = pd.DataFrame(columns=['ForecastId','ConfirmedCases','Fatalities'], dtype='int32')
 df = pd.concat([train,test],sort=False)
 df['Country'] = df.Country.fillna("None")
 df['State'] = df.State.fillna("None")
df_test['winPlacePerc']= 0.0
df_train['Type']= 'Train'
df_test['Type']= 'Test'
print(df_train.shape, df_test.shape )<concatenate>
class BellModel() :
def __init__(self,streak_threshold=7,power=10):
self.streak_threshold = streak_threshold
self.power = power
def __str__(self):
return "BellModel(streak_threshold={},power={},
popt={})".format(
 self.streak_threshold,
self.power,
self.popt 
)
 def func(self,X,a,b,c):
result = a*np.exp(-(((X-b)**2)/(2*(c**2+0.1))))
return result.flatten()
def _find_streak(self, arr):
pos = np.clip(arr, 0, 1 ).astype(bool ).cumsum()
neg = np.clip(arr, -1, 0 ).astype(bool ).cumsum()
streaks = np.where(arr >= 0,
pos-np.maximum.accumulate(np.where(arr <= 0, pos, 0)) ,
-neg+np.maximum.accumulate(np.where(arr >= 0, neg, 0)))
 return streaks
def fit(self,x,y):
self.streak_threshold = -self.streak_threshold
x = scaler.inverse_transform(x)
 y = y.T.flatten() ; x = x.T.flatten()
c = np.argsort(x[:]); y = y[c]
x = np.sort(x)
 nonzero_index, *_ = np.where(np.sign(y ).cumsum() == 1)
 if not isinstance(nonzero_index,np.ndarray)and not nonzero_index:
self.popt = [0,0,0]; return
else:
try:
nonzero_index = nonzero_index.item(0)
 except IndexError:
self.popt = [0,0,0]; return
zeros, y_sp = y[:nonzero_index],y[nonzero_index:]
a = b = d = 0; c = 1
a_min = b_min = 0; c_min = 1
a_min = np.max(y_sp)
 a_max = a_min*self.power+1
d = len(zeros)
 y1 = np.diff(y_sp)
 y1_streaks = self._find_streak(y1)
 if self.streak_threshold in y1_streaks:
a = np.max(y_sp)
 else:
b_min = len(y_sp)+ d
y2 = np.diff(y1)
 y2_streaks = self._find_streak(y2)
 if self.streak_threshold in y2_streaks:
b = np.where(y2_streaks==self.streak_threshold)[0][0] * 2
c = b/3
else:
c_min = len(y_sp)*2/3
a = a_min if a < a_min else a
b = 300 if b < b_min else b
c = 100 if c < c_min else c
try:
self.popt, self.pconv = curve_fit(
 self.func, x, y,
maxfev=100000,
check_finite=False,
p0=[a,b,c],
bounds=([a_min,b_min,c_min],[a_max,300.,100.])
)
 except Exception as e:
traceback.print_exc()
self.popt = [0,0,0]
return self
def get_params(self, deep=False):
return {
'streak_threshold': self.streak_threshold,
'power': self.power,
}
def set_params(self, **params):
for param, value in params.items() :
setattr(self, param, value)
 return self
def get_estimated_params(self):
return [self.streak_threshold, self.power]
def set_curve_params(self, streak_threshold, power):
self.streak_threshold = streak_threshold
self.power = power
return self
def predict(self,x):
x = scaler.inverse_transform(x ).T.flatten()
return self.func(x,*self.popt)
df= pd.concat([df_train, df_test], ignore_index=True )<drop_column>
def model_fit(X_train, Y_train, X_test):

 X_train = scaler.fit_transform(X_train.reshape(-1,1))
X_test = scaler.transform(X_test.reshape(-1,1))
X_train_sh, Y_train_sh = shuffle(X_train,Y_train, random_state=0)
 remove_neg = lambda x: np.rint(x ).astype(int ).clip(min=0)
 metric = "neg_mean_squared_error"
param_grid = {
'streak_threshold': [5,6,7,14,21],
'power': [2,3,5,10,20,30,50]
}
bell_grid = GridSearchCV(BellModel() ,
param_grid,
cv=3,
scoring=metric,
verbose=0 
)
 bell_grid.fit(X_train_sh, Y_train_sh)
 bell_model = bell_grid.best_estimator_
param_grid = {
'polynomialfeatures__degree': np.arange(1,5),
'ridge__alpha':[1e2, 1e3,1e4],
'ridge__fit_intercept': [True, False],
'ridge__normalize': [True, False],
'ridge__solver': ['auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg']
}
def PolynomialRidgeRegression(degree=2, **kwargs):
return make_pipeline(PolynomialFeatures(degree), Ridge(**kwargs))
ridge_grid = GridSearchCV(PolynomialRidgeRegression() ,
param_grid,
cv=3,
scoring=metric,
verbose=0)
 ridge_grid.fit(X_train_sh, Y_train_sh)
 ridge_model = ridge_grid.best_estimator_
param_grid = {
'polynomialfeatures__degree': np.arange(1,5),
'linearsvr__C' : np.logspace(0,1,5),
'linearsvr__epsilon' : np.logspace(-1,1,5),
'linearsvr__fit_intercept': [True, False],
}
def PolynomialSVRRegression(degree=2, **kwargs):
return make_pipeline(PolynomialFeatures(degree), LinearSVR(**kwargs,max_iter=10000))
svr_grid = GridSearchCV(PolynomialSVRRegression() ,
param_grid,
cv=3,
scoring=metric,
verbose=0)
 svr_grid.fit(X_train_sh, Y_train_sh)
 svr_model = svr_grid.best_estimator_
X_train2 = pd.DataFrame({'Bell': remove_neg(bell_model.predict(X_train)) ,
'Ridge': remove_neg(ridge_model.predict(X_train)) ,
'SVR': remove_neg(svr_model.predict(X_train)) ,
})
 X_test2 = pd.DataFrame({ 'Bell': remove_neg(bell_model.predict(X_test)) ,
'Ridge': remove_neg(ridge_model.predict(X_test)) ,
'SVR': remove_neg(svr_model.predict(X_test)) ,
})
 reg = LinearRegression()
reg.fit(X_train2, Y_train)
 Y_test = reg.predict(X_test2)
 Y_test = remove_neg(Y_test)
 return Y_test
df= reduce_mem_usage(df )<feature_engineering>
for name,state in df.groupby(['Country','State']):
mask = state.duplicated(subset=['Date'],keep='first')
 df_tmp = state.loc[~mask]
df_tmp_dropped = state.loc[mask]
drop_num = len(state)- len(df_tmp)
 try:
df_tmp.insert(0,'Index', range(1,len(df_tmp)+1))
except:
pass
tmp_train = df_tmp.dropna(subset=['ConfirmedCases'])
 X_tr = tmp_train['Index'].values
tmp_test = df_tmp[ df_tmp['ConfirmedCases'].isna() ]
X_te = tmp_test.Index.values
for cat in('ConfirmedCases', 'Fatalities'):
Y_tr = tmp_train[cat].values
Y_te = model_fit(X_tr, Y_tr, X_te)
 tmp_test.loc[:,cat] = Y_te
tmp_train = pd.merge(tmp_train.tail(drop_num)[['Date','ConfirmedCases','Fatalities']],
df_tmp_dropped[['Date','ForecastId']],
on='Date')
 col = ['ForecastId','ConfirmedCases','Fatalities']
tmp_train = tmp_train[col]
tmp_test = tmp_test[col]
tmp = pd.concat([tmp_train,tmp_test])
 submission = pd.concat([submission, tmp])
 os.chdir("/kaggle/working/")
 submission = submission.fillna(0)
 submission = submission.astype(int)
 submission.to_csv('submission.csv', index=False)
 plot_data(train,test,submission)