Kaggle competition: COVID19 Global Forecasting (Week 1)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-1)<create_dataframe>
sns.set(context="notebook", style="darkgrid", palette="deep", font="sans-serif", font_scale=1, color_codes=True)
df_model_select = full_table.groupby(['Country/Region','Province/State'] ).last().reset_index()
df_model_select = df_model_select[['Country/Region','Province/State','Deaths']]
df_model_select['Offset error'] = 100
df_model_select['No Offset error'] = 100<data_type_conversions>
data= pd.read_csv(".. /input/covid19-global-forecasting-week-1/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-1/test.csv" )
countries = list(set(full_table['Country/Region']))
countries.sort()
for country in countries:
clusters = list(set(full_table['Province/State'][(full_table['Country/Region'] == country)]))
clusters.sort()
for cluster in clusters:
print(' ')
 print('-----------------')
 print(country + ' - ' + cluster)
 df = full_table[(full_table['Country/Region'] == country)&(full_table['Province/State'] == cluster)]
df = df.groupby(['Date','Country/Region'] ).sum().reset_index()
df['Date'] = pd.to_datetime(df['Date'])
 df = df.sort_values(by=['Date'])
 df = df.set_index('Date')[['Deaths']]
df.drop(df.tail(3 ).index,inplace=True)
 df_result = df.copy()
def model(N, a, alpha, t0, t):
return N *(1 - math.e **(-a *(t-t0)))** alpha
def model_loss(params):
N, a, alpha, t0 = params
global df
r = 0
for t in range(len(df)) :
r +=(model(N, a, alpha, t0, t)- df.iloc[t, 0])** 2
return r
try:
N = df['Deaths'][-1]
T = -df['Deaths'][0]
except:
N = 10000
T = 0
opt = minimize(model_loss, x0=np.array([N, 0.1, 5, T]), method='Nelder-Mead', tol=1e-7 ).x
print('Offset' + str(opt))
x_actual = pd.to_datetime(df.reset_index().iloc[:,0])
 x_actual =list(x_actual)
 y_actual = list(df.reset_index().iloc[:,1])
 start_date = pd.to_datetime(df.index[0])
 x_model = []
y_model = []
for t in range(len(df)+ days_forecast):
x_model.append(start_date + datetime.timedelta(days=t))
y_model.append(round(model(*opt,t)))
 df2 = pd.DataFrame(y_model,index=x_model,columns=['Offset'])
 df2.index.name = 'Date'
df_result = pd.merge(df_result,
df2,
how='outer',
left_on=['Date'],
right_on=['Date'])
 def model(N, a, alpha, t):
return N *(1 - math.e **(-a *(t)))** alpha
def model_loss(params):
N, a, alpha = params
global df
r = 0
for t in range(len(df)) :
r +=(model(N, a, alpha, t)- df.iloc[t, 0])** 2
return r
try:
N = df['Deaths'][-1]
except:
N = 10000
opt = minimize(model_loss, x0=np.array([N, 0.1, 5]), method='Nelder-Mead', tol=1e-7 ).x
print('No offset' + str(opt))
try:
start_date = pd.to_datetime(df.index[0])
 x_model = []
y_model = []
for t in range(len(df)+ days_forecast):
x_model.append(start_date + datetime.timedelta(days=t))
y_model.append(round(model(*opt,t)))
 df2 = pd.DataFrame(y_model,index=x_model,columns=['No Offset'])
 df2.index.name = 'Date'
df_result = pd.merge(df_result,
df2,
how='outer',
left_on=['Date'],
right_on=['Date'])
 df_result = df_result[df_result['Deaths'].notnull() ]
err_offset = 0
err_no_offset = 0
for t in range(len(df_result)) :
err_offset +=(math.log(df_result['Offset'].iloc[t]+1)-math.log(df_result['Deaths'].iloc[t]+1)) **2
err_no_offset +=(math.log(df_result['No Offset'].iloc[t]+1)-math.log(df_result['Deaths'].iloc[t]+1)) **2
err_offset = math.sqrt(err_offset/len(df_result))
err_no_offset = math.sqrt(err_no_offset/len(df_result))
df_model_select['Offset error'][(df_model_select['Country/Region']==country)&(df_model_select['Province/State']==cluster)] = err_offset
df_model_select['No Offset error'][(df_model_select['Country/Region']==country)&(df_model_select['Province/State']==cluster)] = err_no_offset
except:
pass<load_from_csv>
data['Province/State']=data['Province/State'].fillna('' )
df_ca_train = pd.read_csv('.. /input/covid19-global-forecasting-week-1/train.csv')
 df_ca_test = pd.read_csv('.. /input/covid19-global-forecasting-week-1/test.csv')
 df_ca_submission = pd.read_csv('.. /input/covid19-global-forecasting-week-1/submission.csv' )<data_type_conversions>
test['Province/State']=test['Province/State'].fillna('' )
full_table = df_ca_train
full_table[['Province/State']] = full_table[['Province/State']].fillna('')
 df_comp = df_ca_test
df_comp[['Province/State']] = df_comp[['Province/State']].fillna('')
 df_comp['Date'] = pd.to_datetime(df_comp['Date'])
 df_comp['ConfirmedCases']=0
df_comp['Fatalities']=0<data_type_conversions>
data['Province/State'].value_counts()
countries = list(set(full_table['Country/Region']))
countries.sort()
for country in countries:
clusters = list(set(full_table['Province/State'][(full_table['Country/Region'] == country)]))
clusters.sort()
for cluster in clusters:
print(' ')
 print('-----------------')
 print(str(country)+ ' - ' + str(cluster))
df = full_table[(full_table['Country/Region'] == country)&(full_table['Province/State'] == cluster)]
df = df.groupby(['Date','Country/Region'] ).sum().reset_index()
df['Date'] = pd.to_datetime(df['Date'])
 df = df.sort_values(by=['Date'])
 df = df.set_index('Date')[['ConfirmedCases']]
df_result = df.copy()
def model(N, a, alpha, t):
return N *(1 - math.e **(-a *(t)))** alpha
def model_loss(params):
N, a, alpha = params
global df
r = 0
for t in range(len(df)) :
r +=(model(N, a, alpha, t)- df.iloc[t, 0])** 2
return r
try:
N = df['ConfirmedCases'][-1]
except:
N = 10000
opt = minimize(model_loss, x0=np.array([N, 0.1, 5]), method='Nelder-Mead', tol=1e-7 ).x
print(opt)
 x_actual = pd.to_datetime(df.reset_index().iloc[:,0])
 x_actual = list(x_actual)
 y_actual = list(df.reset_index().iloc[:,1])
 start_date = pd.to_datetime(df.index[0])
 days_forecast = len(df)+len(df_ca_test)-7
x_model = []
y_model = []
for t in range(days_forecast):
x_model.append(start_date + datetime.timedelta(days=t))
y_model.append(round(model(*opt,t)))
 df2 = pd.DataFrame(y_model,index=x_model,columns=['ConfirmedCases'])
 df2.index.name = 'Date'
df2['Country/Region']=country
df2['Province/State']=cluster
df_comp = pd.merge(df_comp,
df2,
how='left',
on=['Date','Country/Region','Province/State'] 
)
 df_comp = df_comp.rename(columns={'ConfirmedCases_y': 'ConfirmedCases'})
 df_comp['ConfirmedCases'] = df_comp['ConfirmedCases'].fillna(df_comp['ConfirmedCases_x'])
 df_comp = df_comp[['ForecastId','Province/State','Country/Region','Date','ConfirmedCases','Fatalities']] 
<data_type_conversions>
test['Province/State'].value_counts()
countries = list(set(full_table['Country/Region']))
countries.sort()
for country in countries:
clusters = list(set(full_table['Province/State'][(full_table['Country/Region'] == country)]))
clusters.sort()
for cluster in clusters:
print(' ')
 print('-----------------')
 print(str(country)+ ' - ' + str(cluster))
df = full_table[(full_table['Country/Region'] == country)&(full_table['Province/State'] == cluster)]
df = df.groupby(['Date','Country/Region'] ).sum().reset_index()
df['Date'] = pd.to_datetime(df['Date'])
 df = df.sort_values(by=['Date'])
 df = df.set_index('Date')[['Fatalities']]
df_result = df.copy()
def model(N, a, alpha, t):
return N *(1 - math.e **(-a *(t)))** alpha
def model_loss(params):
N, a, alpha = params
global df
r = 0
for t in range(len(df)) :
r +=(model(N, a, alpha, t)- df.iloc[t, 0])** 2
return r
try:
N = df['Fatalities'][-1]
except:
N = 10000
opt = minimize(model_loss, x0=np.array([N, 0.1, 5]), method='Nelder-Mead', tol=1e-7 ).x
print(opt)
 x_actual = pd.to_datetime(df.reset_index().iloc[:,0])
 x_actual = list(x_actual)
 y_actual = list(df.reset_index().iloc[:,1])
 start_date = pd.to_datetime(df.index[0])
 days_forecast = len(df)+len(df_ca_test)-7
x_model = []
y_model = []
for t in range(days_forecast):
x_model.append(start_date + datetime.timedelta(days=t))
y_model.append(round(model(*opt,t)))
 df2 = pd.DataFrame(y_model,index=x_model,columns=['Fatalities'])
 df2.index.name = 'Date'
df2['Country/Region']=country
df2['Province/State']=cluster
df_comp = pd.merge(df_comp,
df2,
how='left',
on=['Date','Country/Region','Province/State'] 
)
 df_comp = df_comp.rename(columns={'Fatalities_y': 'Fatalities'})
 df_comp['Fatalities'] = df_comp['Fatalities'].fillna(df_comp['Fatalities_x'])
 df_comp = df_comp[['ForecastId','Province/State','Country/Region','Date','ConfirmedCases','Fatalities']] 
<filter>
from sklearn.preprocessing import LabelEncoder
df_comp[df_comp['Fatalities']>0]<data_type_conversions>
le_state=LabelEncoder()
df_ca_test['Date'] = pd.to_datetime(df_ca_test['Date'] )<prepare_output>
le_state=le_state.fit(data['Province/State'] )
df_sub = df_comp[['ForecastId','ConfirmedCases','Fatalities']]<save_to_csv>
data['Province/State']=le_state.transform(data['Province/State'])
 test['Province/State']=le_state.transform(test['Province/State'] )
df_sub.to_csv('submission.csv',index=False )<import_modules>
le_country=LabelEncoder()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
from statsmodels.tsa.statespace.sarimax import SARIMAX
from statsmodels.tsa.arima_model import ARIMA
from random import random<drop_column>
le_country=le_country.fit(data['Country/Region'] )
df=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/train.csv",usecols=['Lat','Province/State','Country/Region','Date','ConfirmedCases','Fatalities'])
 df.fillna(' ',inplace=True)
 df['Lat']=df['Province/State']+df['Country/Region']
df.drop('Province/State',axis=1,inplace=True)
 df.drop('Country/Region',axis=1,inplace=True)
 countries_list=df.Lat.unique()
df1=[]
for i in countries_list:
df1.append(df[df['Lat']==i])
 print("we have "+ str(len(df1)) +" regions in our dataset" )<train_model>
data['Country/Region']=le_country.transform(data['Country/Region'])
 test['Country/Region']=le_country.transform(test['Country/Region'] )
submit_confirmed=[]
submit_fatal=[]
for i in df1:
data = i.ConfirmedCases.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(2,1,0), seasonal_order=(1,1,0,12))
model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_confirmed.extend(list(new[-43:]))
except:
submit_confirmed.extend(list(data[-10:-1]))
for j in range(34):
submit_confirmed.append(data[-1]*2)
 data = i.Fatalities.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(2,1,0), seasonal_order=(1,1,0,12))
model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_fatal.extend(list(new[-43:]))
except:
submit_fatal.extend(list(data[-10:-1]))
for j in range(34):
submit_fatal.append(data[-1]*2)
 df_submit=pd.concat([pd.Series(np.arange(1,1+len(submit_confirmed))),pd.Series(submit_confirmed),pd.Series(submit_fatal)],axis=1 )<save_to_csv>
datetime_str = '01/22/20 00:00:00'
datetime_object = datetime.strptime(datetime_str, '%m/%d/%y %H:%M:%S')
df_submit=df_submit.fillna(method='pad' ).astype(int)
 df_submit.to_csv('submission.csv',header=['ForecastId','ConfirmedCases','Fatalities'],index=False )<set_options>
data['days']=pd.to_datetime(data['Date'] ).sub(datetime_object)/np.timedelta64(1, 'D' )
%matplotlib inline 
<load_from_csv>
test['days']=pd.to_datetime(test['Date'] ).sub(datetime_object)/np.timedelta64(1, 'D' )
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/train.csv')
 train['Date'] = pd.to_datetime(train['Date'])
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv')
 test['Date'] = pd.to_datetime(test['Date'])
 sub = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/submission.csv')
 sub['ForecastId'] = test['ForecastId'].values
sub = sub.set_index('ForecastId', drop=True)
 train<split>
import xgboost as xgb
print(len(train))
eval = train[train["Date"] >= min(test['Date'])]
train = train[train["Date"] < min(test['Date'])]
print(len(train))<compute_train_metric>
data.isna().sum(axis=0)/data.shape[0]
def sigmoid(t, M, beta, alpha):
return M /(1 + np.exp(-beta *(t - alpha)))
 sigmoid_models = {}
def forecast(data, steps, key=None):
if sum(data)== 0 or sum(data != 0)<= 3:
return [data[-1]] * steps
filtered_data = data[data > 0]
try:
def error(params):
M, beta, alpha = params
return np.sqrt(mean_squared_log_error(filtered_data, sigmoid(list(range(sum(data == 0), len(data))), M, beta, alpha)))
 res = opt.minimize(error, x0=[1000, 0.25, 100])
 M, beta, alpha = res.x
sigmoid_models[key] =(M, beta, alpha)
 return np.clip(sigmoid(list(range(len(data), len(data)+ steps)) , M, beta, alpha), 0, None ).astype(int)
 except:
try:
popt, pcov = opt.curve_fit(sigmoid, list(range(sum(data == 0), len(data))),
filtered_data, maxfev=10000, p0=[1000, 0.25, 100])
 M, beta, alpha = popt
sigmoid_models[key] =(M, beta, alpha)
 return np.clip(sigmoid(list(range(len(data), len(data)+ steps)) , M, beta, alpha), 0, None ).astype(int)
 except:
print(key)
 return [data[-1]] * steps
for country in tqdm_notebook(np.unique(train['Country/Region'])) :
country_data = train[(train['Country/Region'] == country)]
country_data = country_data.sort_values(by='Date')
 if country not in ['US', 'China', 'Canada', 'Australia', 'France', 'United Kingdom', 'Netherlands', 'Denmark', 'Cruise Ship']:
country_test_data = test[test['Country/Region'] == country].sort_values(by='Date')
 sub.loc[country_test_data['ForecastId'], 'ConfirmedCases'] = forecast(country_data['ConfirmedCases'].values, len(country_test_data), key=(country, np.NaN, 'ConfirmedCases'))
sub.loc[country_test_data['ForecastId'], 'Fatalities'] = forecast(country_data['Fatalities'].values, len(country_test_data), key=(country, np.NaN, 'Fatalities'))
else:
for region in np.unique(country_data['Province/State']):
region_data = country_data[country_data['Province/State'] == region]
region_data = region_data.sort_values(by='Date')
 region_test_data = test[(test['Country/Region'] == country)&(test['Province/State'] == region)].sort_values(by='Date')
 sub.loc[region_test_data['ForecastId'], 'ConfirmedCases'] = forecast(region_data['ConfirmedCases'].values, len(region_test_data), key=(country, region, 'ConfirmedCases'))
sub.loc[region_test_data['ForecastId'], 'Fatalities'] = forecast(region_data['Fatalities'].values, len(region_test_data), key=(country, region, 'Fatalities'))<merge>
test.isna().sum(axis=0 )
eval1 = eval.merge(test, how='left', on=['Date', 'Country/Region', 'Province/State'])
 eval1 = eval1.merge(sub, left_on=['ForecastId'], right_index=True)
 score_c = np.sqrt(mean_squared_log_error(eval1['ConfirmedCases_x'].values, eval1['ConfirmedCases_y']))
score_f = np.sqrt(mean_squared_log_error(eval1['Fatalities_x'].values, eval1['Fatalities_y']))
print(f'score_c: {score_c}, score_f: {score_f}, mean: {np.mean([score_c, score_f])}' )<save_to_csv>
x = data[['Province/State','Country/Region','Lat', 'Long', 'days']]
y1 = data['ConfirmedCases']
y2 = data['Fatalities']
x_test_set = test[['Province/State','Country/Region','Lat', 'Long', 'days']]
sub.to_csv('submission.csv' )<load_from_csv>
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 train_data.head()<data_type_conversions>
model=xgb.XGBRegressor(base_score=0.5, booster=None, colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0.1, gpu_id=-1,
importance_type='gain', interaction_constraints=None,
learning_rate=0.1, max_delta_step=0, max_depth=20,
min_child_weight=1, monotone_constraints=None,
n_estimators=2048, n_jobs=0, num_parallel_tree=10,
objective='reg:squarederror', random_state=0, reg_alpha=0,
reg_lambda=1, scale_pos_weight=1, subsample=1, tree_method=None,
validate_parameters=False, verbosity=None )
avg_mean = train_data["Age"].astype('float' ).mean(axis=0 )<drop_column>
model.fit(x,y1 )
train_data.drop("Cabin", axis = 1, inplace=True )<drop_column>
pred1 = model.predict(x_test_set )
train_data.drop("Name", axis = 1, inplace=True )<define_variables>
pred1=np.array([int(round(p)) for p in pred1] )
train_data["Embarked"].replace(np.nan, 'S', inplace=True )<data_type_conversions>
pred1[np.array(pred1)<0]=0
train_data["Age"] = train_data["Age"].astype("int" )<load_from_csv>
model.fit(x,y2)
 pred2 = model.predict(x_test_set)
 pred2=np.array([int(round(p)) for p in pred2])
 pred2[pred2<0]=0
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data.head()<data_type_conversions>
Sub = pd.read_csv(".. /input/covid19-global-forecasting-week-1/submission.csv")
 Sub.columns
sub_new = Sub["ForecastId"]
avg_mean = test_data["Age"].astype('float' ).mean(axis=0)
 test_data["Age"].replace(np.nan, avg_mean, inplace=True)
 test_data["Age"].astype("int")
 Favg_mean = test_data["Fare"].astype('float' ).mean(axis=0)
 test_data["Fare"].replace(np.nan, Favg_mean, inplace=True )<define_search_space>
OP = pd.DataFrame()
OP['ForecastId']=Sub["ForecastId"]
OP['ConfirmedCases']=pred1
OP['Fatalities']=pred2
parameters = {
"n_estimators":[5,10,50,100,250],
"max_depth":[2,4,8,16,32,None]
}<prepare_x_and_y>
OP["ConfirmedCases"] = OP["ConfirmedCases"].astype(int)
 OP["Fatalities"] = OP["Fatalities"].astype(int)
lm =RandomForestClassifier(n_estimators=250, max_depth=8, random_state=1)
 y = train_data["Survived"]
features = ["Pclass", "Sex", "Age", "SibSp", "Parch","Fare","Embarked"]
X = pd.get_dummies(train_data[features])
 X_test = pd.get_dummies(test_data[features])
 lm.fit(X, train_data['Survived'] )<save_to_csv>
OP.to_csv("submission.csv",index=False )