Kaggle competition: COVID19 Global Forecasting (Week 1)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-1)<feature_engineering>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from IPython.core.display import display, HTML
import plotly.graph_objects as go
import warnings
import datetime
import math
from scipy.optimize import minimize
train_data["Title"] = train_data["Name"].str.split(",", expand=True)[1].str.split(".", expand=True)[0]
test_data["Title"] = test_data["Name"].str.split(",", expand=True)[1].str.split(".", expand=True)[0]
title_counts = train_data["Title"].value_counts() < 10
train_data["Title"] = train_data["Title"].apply(lambda x: "Gen" if title_counts.loc[x] else x)
 title_counts_tst = test_data["Title"].value_counts() < 10
test_data["Title"] = test_data["Title"].apply(lambda x: "Gen" if title_counts_tst.loc[x] else x )<data_type_conversions>
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', 500)
 pd.set_option('display.expand_frame_repr', False)
 display(HTML("<style>div.output_scroll { height: 35em; }</style>"))
%matplotlib inline
%config InlineBackend.figure_format ='retina'
warnings.filterwarnings('ignore' )
train_data["Marry_Status"] = 0
train_data["Marry_Status"].loc[train_data["Title"].str.strip() == "Mrs"] = 1
train_data["Marry_Status"] = train_data["Marry_Status"].astype("category")
 test_data["Marry_Status"] = 0
test_data["Marry_Status"].loc[test_data["Title"].str.strip() == "Mrs"] = 1
test_data["Marry_Status"] = test_data["Marry_Status"].astype("category" )<data_type_conversions>
days_forecast = 40
train_data["Pclass"] = train_data["Pclass"].astype("category")
 test_data["Pclass"] = test_data["Pclass"].astype("category")
 y = train_data["Survived"]<categorify>
conf_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Confirmed.csv')
 deaths_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Deaths.csv')
 recv_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Recovered.csv' )
features = ["Pclass", "Sex", "Age", "SibSp", "Parch", "Fare", "Cabin", "Embarked", "Family_Size",
"Travelled_Together", "Title", "Marry_Status"]
X = pd.get_dummies(train_data[features])
 X_test = pd.get_dummies(test_data[features] )<choose_model_class>
full_table = full_table[full_table['Province/State'].str.contains(',')!=True]
chosed_model = "rf"
if chosed_model == "dp":
model = Sequential()
model.add(Dense(10, input_dim=36, activation='tanh'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.fit(X, y, epochs=100, batch_size=5, verbose=0)
 predictions = model.predict_classes(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions.flatten() })
 if(chosed_model=="rf")|(chosed_model == "xboost"):
model = RandomForestClassifier(n_estimators=200, max_depth=7, random_state=1)
 model.fit(X, y)
 predictions = model.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('/kaggle/working/my_submission_rf.csv', index=False)
 if(chosed_model == "xboost")|(chosed_model =="rf"):
gbm = xgb.XGBClassifier(
 n_estimators=2000,
max_depth=4,
min_child_weight=2,
gamma=0.9,
subsample=0.8,
colsample_bytree=0.8,
objective='binary:logistic',
nthread=-1,
scale_pos_weight=1 ).fit(X, y)
 xgb_predictions = gbm.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('/kaggle/working/my_submission_xboost.csv', index=False)
 if(chosed_model == 'knn'):
knn = KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',
metric_params=None, n_jobs=1, n_neighbors=18, p=2,
weights='distance')
 knn.fit(X, y)
 knn_predictions = knn.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': knn_predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!")
<save_to_csv>
cases = ['Confirmed', 'Deaths', 'Recovered', 'Active']
full_table['Active'] = full_table['Confirmed'] - full_table['Deaths'] - full_table['Recovered']
full_table['Country/Region'] = full_table['Country/Region'].replace('Mainland China', 'China')
 full_table[['Province/State']] = full_table[['Province/State']].fillna('' )
if(chosed_model=="rf")|(chosed_model == "xboost"):
df_xboost = pd.read_csv('/kaggle/working/my_submission_xboost.csv')
 df_rf = pd.read_csv('/kaggle/working/my_submission_rf.csv')
 df_ensemble = df_xboost.copy()
k = 'Survived'
df_ensemble[k] = 0.5 * df_xboost[k] + 0.7 * df_rf[k]
df_ensemble[k] = df_ensemble[k].apply(lambda f: 1 if f>=0.5 else 0)
 df_ensemble.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<set_options>
df = full_table.groupby(['Country/Region', 'Province/State'])['Confirmed', 'Deaths', 'Recovered', 'Active'].max()
df = full_table.groupby('Date')['Confirmed', 'Deaths', 'Recovered', 'Active'].sum().reset_index()
df = df[df['Date']==max(df['Date'])].reset_index(drop=True)
 df
%matplotlib inline 
<define_variables>
df_model_select = full_table.groupby(['Country/Region','Province/State'] ).last().reset_index()
df_model_select = df_model_select[['Country/Region','Province/State','Confirmed']]
df_model_select['Offset error'] = 100
df_model_select['No Offset error'] = 100
DATA_PATH = '.. /input'
SUBMISSIONS_PATH = './'
ATOMIC_NUMBERS = {
'H': 1,
'C': 6,
'N': 7,
'O': 8,
'F': 9
}<set_options>
countries = list(set(full_table['Country/Region']))
countries.sort()
for country in countries:
clusters = list(set(full_table['Province/State'][(full_table['Country/Region'] == country)]))
clusters.sort()
for cluster in clusters:
print(' ')
 print('-----------------')
 print(country + ' - ' + cluster)
 df = full_table[(full_table['Country/Region'] == country)&(full_table['Province/State'] == cluster)]
df = df.groupby(['Date','Country/Region'] ).sum().reset_index()
df['Date'] = pd.to_datetime(df['Date'])
 df = df.sort_values(by=['Date'])
 df = df.set_index('Date')[['Confirmed']]
df.drop(df.tail(3 ).index,inplace=True)
 df_result = df.copy()
def model(N, a, alpha, t0, t):
return N *(1 - math.e **(-a *(t-t0)))** alpha
def model_loss(params):
N, a, alpha, t0 = params
global df
r = 0
for t in range(len(df)) :
r +=(model(N, a, alpha, t0, t)- df.iloc[t, 0])** 2
return r
try:
N = df['Confirmed'][-1]
T = -df['Confirmed'][0]
except:
N = 10000
T = 0
opt = minimize(model_loss, x0=np.array([N, 1.5, 5, T]), method='Nelder-Mead', tol=1e-7 ).x
print('Offset' + str(opt))
x_actual = pd.to_datetime(df.reset_index().iloc[:,0])
 x_actual =list(x_actual)
 y_actual = list(df.reset_index().iloc[:,1])
 start_date = pd.to_datetime(df.index[0])
 x_model = []
y_model = []
for t in range(len(df)+ days_forecast):
x_model.append(start_date + datetime.timedelta(days=t))
y_model.append(round(model(*opt,t)))
 df2 = pd.DataFrame(y_model,index=x_model,columns=['Offset'])
 df2.index.name = 'Date'
df_result = pd.merge(df_result,
df2,
how='outer',
left_on=['Date'],
right_on=['Date'])
 def model(N, a, alpha, t):
return N *(1 - math.e **(-a *(t)))** alpha
def model_loss(params):
N, a, alpha = params
global df
r = 0
for t in range(len(df)) :
r +=(model(N, a, alpha, t)- df.iloc[t, 0])** 2
return r
try:
N = df['Confirmed'][-1]
except:
N = 10000
opt = minimize(model_loss, x0=np.array([N, 1.5, 5]), method='Nelder-Mead', tol=1e-7 ).x
print('No offset' + str(opt))
try:
start_date = pd.to_datetime(df.index[0])
 x_model = []
y_model = []
for t in range(len(df)+ days_forecast):
x_model.append(start_date + datetime.timedelta(days=t))
y_model.append(round(model(*opt,t)))
 df2 = pd.DataFrame(y_model,index=x_model,columns=['No Offset'])
 df2.index.name = 'Date'
df_result = pd.merge(df_result,
df2,
how='outer',
left_on=['Date'],
right_on=['Date'])
 df_result = df_result[df_result['Confirmed'].notnull() ]
err_offset = 0
err_no_offset = 0
for t in range(len(df_result)) :
err_offset +=(math.log(df_result['Offset'].iloc[t]+1)-math.log(df_result['Confirmed'].iloc[t]+1)) **2
err_no_offset +=(math.log(df_result['No Offset'].iloc[t]+1)-math.log(df_result['Confirmed'].iloc[t]+1)) **2
err_offset = math.sqrt(err_offset/len(df_result))
err_no_offset = math.sqrt(err_no_offset/len(df_result))
df_model_select['Offset error'][(df_model_select['Country/Region']==country)&(df_model_select['Province/State']==cluster)] = err_offset
df_model_select['No Offset error'][(df_model_select['Country/Region']==country)&(df_model_select['Province/State']==cluster)] = err_no_offset
except:
pass
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 120)
 pd.set_option('display.max_columns', 120 )<load_from_csv>
df_model_select = full_table.groupby(['Country/Region','Province/State'] ).last().reset_index()
df_model_select = df_model_select[['Country/Region','Province/State','Deaths']]
df_model_select['Offset error'] = 100
df_model_select['No Offset error'] = 100
train_dtypes = {
'molecule_name': 'category',
'atom_index_0': 'int8',
'atom_index_1': 'int8',
'type': 'category',
'scalar_coupling_constant': 'float32'
}
train_csv = pd.read_csv(f'{DATA_PATH}/train.csv', index_col='id', dtype=train_dtypes)
 train_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 train_csv = train_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type', 'scalar_coupling_constant']]
train_csv.head(10 )<load_from_csv>
countries = list(set(full_table['Country/Region']))
countries.sort()
for country in countries:
clusters = list(set(full_table['Province/State'][(full_table['Country/Region'] == country)]))
clusters.sort()
for cluster in clusters:
print(' ')
 print('-----------------')
 print(country + ' - ' + cluster)
 df = full_table[(full_table['Country/Region'] == country)&(full_table['Province/State'] == cluster)]
df = df.groupby(['Date','Country/Region'] ).sum().reset_index()
df['Date'] = pd.to_datetime(df['Date'])
 df = df.sort_values(by=['Date'])
 df = df.set_index('Date')[['Deaths']]
df.drop(df.tail(3 ).index,inplace=True)
 df_result = df.copy()
def model(N, a, alpha, t0, t):
return N *(1 - math.e **(-a *(t-t0)))** alpha
def model_loss(params):
N, a, alpha, t0 = params
global df
r = 0
for t in range(len(df)) :
r +=(model(N, a, alpha, t0, t)- df.iloc[t, 0])** 2
return r
try:
N = df['Deaths'][-1]
T = -df['Deaths'][0]
except:
N = 10000
T = 0
opt = minimize(model_loss, x0=np.array([N, 1.5, 5, T]), method='Nelder-Mead', tol=1e-7 ).x
print('Offset' + str(opt))
x_actual = pd.to_datetime(df.reset_index().iloc[:,0])
 x_actual =list(x_actual)
 y_actual = list(df.reset_index().iloc[:,1])
 start_date = pd.to_datetime(df.index[0])
 x_model = []
y_model = []
for t in range(len(df)+ days_forecast):
x_model.append(start_date + datetime.timedelta(days=t))
y_model.append(round(model(*opt,t)))
 df2 = pd.DataFrame(y_model,index=x_model,columns=['Offset'])
 df2.index.name = 'Date'
df_result = pd.merge(df_result,
df2,
how='outer',
left_on=['Date'],
right_on=['Date'])
 def model(N, a, alpha, t):
return N *(1 - math.e **(-a *(t)))** alpha
def model_loss(params):
N, a, alpha = params
global df
r = 0
for t in range(len(df)) :
r +=(model(N, a, alpha, t)- df.iloc[t, 0])** 2
return r
try:
N = df['Deaths'][-1]
except:
N = 10000
opt = minimize(model_loss, x0=np.array([N, 1.5, 5]), method='Nelder-Mead', tol=1e-7 ).x
print('No offset' + str(opt))
try:
start_date = pd.to_datetime(df.index[0])
 x_model = []
y_model = []
for t in range(len(df)+ days_forecast):
x_model.append(start_date + datetime.timedelta(days=t))
y_model.append(round(model(*opt,t)))
 df2 = pd.DataFrame(y_model,index=x_model,columns=['No Offset'])
 df2.index.name = 'Date'
df_result = pd.merge(df_result,
df2,
how='outer',
left_on=['Date'],
right_on=['Date'])
 df_result = df_result[df_result['Deaths'].notnull() ]
err_offset = 0
err_no_offset = 0
for t in range(len(df_result)) :
err_offset +=(math.log(df_result['Offset'].iloc[t]+1)-math.log(df_result['Deaths'].iloc[t]+1)) **2
err_no_offset +=(math.log(df_result['No Offset'].iloc[t]+1)-math.log(df_result['Deaths'].iloc[t]+1)) **2
err_offset = math.sqrt(err_offset/len(df_result))
err_no_offset = math.sqrt(err_no_offset/len(df_result))
df_model_select['Offset error'][(df_model_select['Country/Region']==country)&(df_model_select['Province/State']==cluster)] = err_offset
df_model_select['No Offset error'][(df_model_select['Country/Region']==country)&(df_model_select['Province/State']==cluster)] = err_no_offset
except:
pass
submission_csv = pd.read_csv(f'{DATA_PATH}/sample_submission.csv', index_col='id' )<load_from_csv>
df_ca_train = pd.read_csv('.. /input/covid19-global-forecasting-week-1/train.csv')
 df_ca_test = pd.read_csv('.. /input/covid19-global-forecasting-week-1/test.csv')
 df_ca_submission = pd.read_csv('.. /input/covid19-global-forecasting-week-1/submission.csv' )
test_csv = pd.read_csv(f'{DATA_PATH}/test.csv', index_col='id', dtype=train_dtypes)
 test_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '' ).astype('int32')
 test_csv = test_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type']]
test_csv.head(10 )<data_type_conversions>
full_table = df_ca_train
full_table[['Province/State']] = full_table[['Province/State']].fillna('')
 df_comp = df_ca_test
df_comp[['Province/State']] = df_comp[['Province/State']].fillna('')
 df_comp['Date'] = pd.to_datetime(df_comp['Date'])
 df_comp['ConfirmedCases']=0
df_comp['Fatalities']=0
structures_dtypes = {
'molecule_name': 'category',
'atom_index': 'int8',
'atom': 'category',
'x': 'float32',
'y': 'float32',
'z': 'float32'
}
structures_csv = pd.read_csv(f'{DATA_PATH}/structures.csv', dtype=structures_dtypes)
 structures_csv['molecule_index'] = structures_csv.molecule_name.str.replace('dsgdb9nsd_', '' ).astype('int32')
 structures_csv = structures_csv[['molecule_index', 'atom_index', 'atom', 'x', 'y', 'z']]
structures_csv['atom'] = structures_csv['atom'].replace(ATOMIC_NUMBERS ).astype('int8')
 structures_csv.head(10 )<data_type_conversions>
countries = list(set(full_table['Country/Region']))
countries.sort()
for country in countries:
clusters = list(set(full_table['Province/State'][(full_table['Country/Region'] == country)]))
clusters.sort()
for cluster in clusters:
print(' ')
 print('-----------------')
 print(str(country)+ ' - ' + str(cluster))
df = full_table[(full_table['Country/Region'] == country)&(full_table['Province/State'] == cluster)]
df = df.groupby(['Date','Country/Region'] ).sum().reset_index()
df['Date'] = pd.to_datetime(df['Date'])
 df = df.sort_values(by=['Date'])
 df = df.set_index('Date')[['ConfirmedCases']]
df_result = df.copy()
def model(N, a, alpha, t):
return N *(1 - math.e **(-a *(t)))** alpha
def model_loss(params):
N, a, alpha = params
global df
r = 0
for t in range(len(df)) :
r +=(model(N, a, alpha, t)- df.iloc[t, 0])** 2
return r
try:
N = df['ConfirmedCases'][-1]
except:
N = 10000
opt = minimize(model_loss, x0=np.array([N, 1.5, 5]), method='Nelder-Mead', tol=1e-7 ).x
print(opt)
 x_actual = pd.to_datetime(df.reset_index().iloc[:,0])
 x_actual = list(x_actual)
 y_actual = list(df.reset_index().iloc[:,1])
 start_date = pd.to_datetime(df.index[0])
 days_forecast = len(df)+len(df_ca_test)-7
x_model = []
y_model = []
for t in range(days_forecast):
x_model.append(start_date + datetime.timedelta(days=t))
y_model.append(round(model(*opt,t)))
 df2 = pd.DataFrame(y_model,index=x_model,columns=['ConfirmedCases'])
 df2.index.name = 'Date'
df2['Country/Region']=country
df2['Province/State']=cluster
df_comp = pd.merge(df_comp,
df2,
how='left',
on=['Date','Country/Region','Province/State'] 
)
 df_comp = df_comp.rename(columns={'ConfirmedCases_y': 'ConfirmedCases'})
 df_comp['ConfirmedCases'] = df_comp['ConfirmedCases'].fillna(df_comp['ConfirmedCases_x'])
 df_comp = df_comp[['ForecastId','Province/State','Country/Region','Date','ConfirmedCases','Fatalities']]
def build_type_dataframes(base, structures, coupling_type):
base = base[base['type'] == coupling_type].drop('type', axis=1 ).copy()
base = base.reset_index()
base['id'] = base['id'].astype('int32')
 structures = structures[structures['molecule_index'].isin(base['molecule_index'])]
return base, structures<merge>
countries = list(set(full_table['Country/Region']))
countries.sort()
for country in countries:
clusters = list(set(full_table['Province/State'][(full_table['Country/Region'] == country)]))
clusters.sort()
for cluster in clusters:
print(' ')
 print('-----------------')
 print(str(country)+ ' - ' + str(cluster))
df = full_table[(full_table['Country/Region'] == country)&(full_table['Province/State'] == cluster)]
df = df.groupby(['Date','Country/Region'] ).sum().reset_index()
df['Date'] = pd.to_datetime(df['Date'])
 df = df.sort_values(by=['Date'])
 df = df.set_index('Date')[['Fatalities']]
df_result = df.copy()
def model(N, a, alpha, t):
return N *(1 - math.e **(-a *(t)))** alpha
def model_loss(params):
N, a, alpha = params
global df
r = 0
for t in range(len(df)) :
r +=(model(N, a, alpha, t)- df.iloc[t, 0])** 2
return r
try:
N = df['Fatalities'][-1]
except:
N = 10000
opt = minimize(model_loss, x0=np.array([N, 1.5, 5]), method='Nelder-Mead', tol=1e-7 ).x
print(opt)
 x_actual = pd.to_datetime(df.reset_index().iloc[:,0])
 x_actual = list(x_actual)
 y_actual = list(df.reset_index().iloc[:,1])
 start_date = pd.to_datetime(df.index[0])
 days_forecast = len(df)+len(df_ca_test)-7
x_model = []
y_model = []
for t in range(days_forecast):
x_model.append(start_date + datetime.timedelta(days=t))
y_model.append(round(model(*opt,t)))
 df2 = pd.DataFrame(y_model,index=x_model,columns=['Fatalities'])
 df2.index.name = 'Date'
df2['Country/Region']=country
df2['Province/State']=cluster
df_comp = pd.merge(df_comp,
df2,
how='left',
on=['Date','Country/Region','Province/State'] 
)
 df_comp = df_comp.rename(columns={'Fatalities_y': 'Fatalities'})
 df_comp['Fatalities'] = df_comp['Fatalities'].fillna(df_comp['Fatalities_x'])
 df_comp = df_comp[['ForecastId','Province/State','Country/Region','Date','ConfirmedCases','Fatalities']]
def add_coordinates(base, structures, index):
df = pd.merge(base, structures, how='inner',
left_on=['molecule_index', f'atom_index_{index}'],
right_on=['molecule_index', 'atom_index'] ).drop(['atom_index'], axis=1)
 df = df.rename(columns={
'atom': f'atom_{index}',
'x': f'x_{index}',
'y': f'y_{index}',
'z': f'z_{index}'
})
 return df<merge>
df_comp[df_comp['Fatalities']>0]
def add_atoms(base, atoms):
df = pd.merge(base, atoms, how='inner',
on=['molecule_index', 'atom_index_0', 'atom_index_1'])
 return df<merge>
df_ca_test['Date'] = pd.to_datetime(df_ca_test['Date'] )
def merge_all_atoms(base, structures):
df = pd.merge(base, structures, how='left',
left_on=['molecule_index'],
right_on=['molecule_index'])
 df = df[(df.atom_index_0 != df.atom_index)&(df.atom_index_1 != df.atom_index)]
return df<feature_engineering>
df_sub = df_comp[['ForecastId','ConfirmedCases','Fatalities']]
<compute_test_metric><EOS> 
df_sub.to_csv('submission.csv',index=False )