Kaggle competition: COVID19 Global Forecasting (Week 3)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-week-3<drop_column>
warnings.filterwarnings('ignore' )
print("fill blanks and add region for counting")
 df.fillna(' ',inplace=True)
 df['Lat']=df['Province_State']+df['Country_Region']
df.drop('Province_State',axis=1,inplace=True)
 df.drop('Country_Region',axis=1,inplace=True)
<load_from_csv>
dftrain = pd.read_csv('.. /input/covid19-global-forecasting-week-3/train.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'] ).fillna('None')
 dftest = pd.read_csv('.. /input/covid19-global-forecasting-week-3/test.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'] ).fillna('None')
 dfsubm = pd.read_csv('.. /input/covid19-global-forecasting-week-3/submission.csv' )
countries_list=df.Lat.unique()
df1=[]
for i in countries_list:
df1.append(df[df['Lat']==i])
 print("we have "+ str(len(df1)) +" regions in our dataset")
 test=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv" )<choose_model_class>
confirmed = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv' ).sort_values(by='Country/Region')
 deaths = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_global.csv')
 recovered = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_recovered_global.csv' )
submit_confirmed=[]
submit_fatal=[]
for i in df1:
data = i.ConfirmedCases.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(1,1,0), seasonal_order=(1,1,0,12),measurement_error=True)
 model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_confirmed.extend(list(new[-43:]))
except:
submit_confirmed.extend(list(data[-10:-1]))
for j in range(34):
submit_confirmed.append(data[-1]*2)
 data = i.Fatalities.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(1,1,0), seasonal_order=(1,1,0,12),measurement_error=True)
 model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_fatal.extend(list(new[-43:]))
except:
submit_fatal.extend(list(data[-10:-1]))
for j in range(34):
submit_fatal.append(data[-1]*2)
<data_type_conversions>
def change_countryprovince(df):
df['Country_Region'] = df['Country/Region']
df['Province_State'] = df['Province/State']
df = df.drop(['Country/Region', 'Province/State'], axis=1)
 return df
df_submit=pd.concat([pd.Series(np.arange(1,1+len(submit_confirmed))),pd.Series(submit_confirmed),pd.Series(submit_fatal)],axis=1)
 df_submit=df_submit.fillna(method='pad' ).astype(int )<merge>
confirmed = change_countryprovince(confirmed)
 deaths = change_countryprovince(deaths)
 recovered = change_countryprovince(recovered)
 csse_data = pd.DataFrame(confirmed.set_index(['Province_State','Country_Region'] ).drop(['Lat','Long'],axis=1 ).stack() ).reset_index()
csse_data.columns = ['Province_State', 'Country_Region', 'Date', 'ConfirmedCases']
csse_data['Fatalities'] = deaths.set_index(['Province_State','Country_Region'] ).drop(['Lat','Long'],axis=1 ).stack().tolist()
csse_data['Date'] = pd.to_datetime(csse_data['Date'])
 csse_data.head()
df_submit.rename(columns={0: 'ForecastId', 1: 'ConfirmedCases',2: 'Fatalities',}, inplace=True)
 complete_test= pd.merge(test, df_submit, how="left", on="ForecastId" )<save_to_csv>
len(csse_data['Province_State'].unique() )
df_submit.to_csv('submission.csv',header=['ForecastId','ConfirmedCases','Fatalities'],index=False)
 complete_test.to_csv('complete_test.csv',index=False)
<import_modules>
len(dftrain['Province_State'].unique() )
plotly.offline.init_notebook_mode()
%matplotlib inline
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<load_from_csv>
dftrain = dftrain.join(confirmed[['Country_Region', 'Province_State', 'Lat', 'Long']].set_index(['Province_State', 'Country_Region']), on=['Province_State', 'Country_Region'] )
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )<categorify>
dftrain['Dayofyear'] = dftrain['Date'].dt.dayofyear
dftest['Dayofyear'] = dftest['Date'].dt.dayofyear
train[['ConfirmedCases', 'Fatalities']] = train.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases', 'Fatalities']].transform('cummax')
<feature_engineering>
def transpose_df(df):
df = df.drop(['Lat','Long'],axis=1 ).groupby('Country_Region' ).sum().T
df.index = pd.to_datetime(df.index)
 return df
feature_day = [1,2,5,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
adjusted_X_pred = X_pred[feature_use].values.reshape(-1, 1)
 model = make_pipeline(PolynomialFeatures(2), BayesianRidge())
 model.fit(adjusted_X_train,adjusted_y_train_confirmed)
 y_hat_confirmed = model.predict(adjusted_X_pred)
 model.fit(adjusted_X_train,adjusted_y_train_fatalities)
 y_hat_fatalities = model.predict(adjusted_X_pred)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_1 = df_val.copy()<compute_test_metric>
confirmedT = transpose_df(confirmed)
 deathsT = transpose_df(deaths)
 recoveredT = transpose_df(recovered)
 mortalityT = deathsT/confirmedT
RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values )<compute_test_metric>
def add_day(df):
df['Date'] = df.index
df['Dayofyear'] = df['Date'].dt.dayofyear
return df
RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values )<groupby>
confirmedT, deathsT, recoveredT, mortalityT = add_day(confirmedT), add_day(deathsT), add_day(recoveredT), add_day(mortalityT )
country = "Vietnam"
df_val = df_val_1
df_val[df_val['Country_Region'] == country].groupby(['Date','Country_Region'] ).sum().reset_index()<save_model>
allcountries_ordered = confirmed.set_index(['Country_Region'] ).iloc[:,-2].sort_values(ascending=False ).index.tolist()
animator.save('confirm_animation.gif', writer='imagemagick', fps=2)
 display(Image(url='confirm_animation.gif'))<feature_engineering>
confirmed.set_index(['Country_Region'] ).iloc[:,-2].sort_values(ascending=False ).to_csv('confirmed_countries.csv' )
feature_day = [1,2,5,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = ExponentialSmoothing(adjusted_y_train_confirmed, trend = 'additive' ).fit()
y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = ExponentialSmoothing(adjusted_y_train_fatalities, trend = 'additive' ).fit()
y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_2 = df_val.copy()<feature_engineering>
def df_day1(df, confirmed):
df_day1 = pd.DataFrame({'Days since 100 cases' : np.arange(1000)} ).set_index('Days since 100 cases')
 countries_df = df.columns.tolist() [:-2]
countries_conf = confirmed.columns.tolist() [:-2]
for ic, country in enumerate(countries_df):
for ic2, country2 in enumerate(countries_conf):
if country == country2:
dfsub = df[confirmed[country] > 100.][country]
df_day1[country] = np.nan
df_day1.loc[:len(dfsub)-1,country] =(dfsub ).tolist()
df_day1 = df_day1.dropna(how='all')
 return df_day1
feature_day = [1,2,5,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()<compute_test_metric>
confirmed_day1 = df_day1(confirmedT, confirmedT)
 deaths_day1 = df_day1(deathsT, confirmedT)
 recovered_day1 = df_day1(recoveredT, confirmedT)
 mortality_day1 = df_day1(mortalityT, confirmedT)
 confirmednorm_day1 = confirmed_day1/confirmed_day1.loc[0,:]
maxday = confirmed_day1.shape[0]
method_list = ['Poly Bayesian Ridge','Exponential Smoothing','SARIMA']
method_val = [df_val_1,df_val_2,df_val_3]
for i in range(0,3):
df_val = method_val[i]
method_score = [method_list[i]] + [RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values)] + [RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values)]
print(method_score )<save_to_csv>
date_day1 = confirmedT.copy()
for column in date_day1:
date_day1[column] = confirmedT.index.tolist()
date_day1 = df_day1(date_day1, confirmedT )
df_val = df_val_3
df_val['ConfirmedCases_hat'] =(df_val_1['ConfirmedCases_hat'] + df_val_2['ConfirmedCases_hat'] + df_val_3['ConfirmedCases_hat'])/3
df_val['Fatalities_hat'] =(df_val_1['Fatalities_hat'] + df_val_2['Fatalities_hat'] + df_val_3['Fatalities_hat'])/3
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission = submission.round({'ConfirmedCases': 0, 'Fatalities': 0})
 submission.to_csv('submission.csv', index=False)
 submission<filter>
def logistic_curve(x, k, x_0, ymax):
return ymax /(1 + np.exp(-k*(x-x_0)) )
df_worldinfor[df_worldinfor['Country'] == 'Vietnam']<import_modules>
def logistic_curve2(x, k1, k2, x_0, ymax):
return ymax /(1 + np.exp(-k1*(x-x_0)) + np.exp(-k2*(x-x_0)) )
print("Read in libraries")
<load_from_csv>
list_countries = dftrain[dftrain['Date'] == '2020-01-22']['Country_Region'].tolist()
list_states = dftrain[dftrain['Date'] == '2020-01-22']['Province_State'].tolist()
datenow = datetime.now()
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 sub = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/submission.csv")
 print("read in files")
 test.head()<concatenate>
list_date_pand = [] ; list_maxcases = []; list_maxfat = []
for country, state in list(zip(list_countries, list_states)) :
df2 = dftrain.loc[(dftrain['Country_Region'] == country)&(dftrain['Province_State'] == state)].fillna('None')
 maxcases, maxfat = df2['ConfirmedCases'].max() , df2['Fatalities'].max()
date_pand2 = []
date_pand = df2[df2['ConfirmedCases'] > 100.]['Date'].tolist()
try:
list_date_pand.append(pd.to_datetime(date_pand[0]))
except:
list_date_pand.append(pd.to_datetime(datenow))
list_maxcases.append(maxcases); list_maxfat.append(maxfat)
 dfstartpand = pd.DataFrame(np.array([list_countries, list_states, list_date_pand, list_maxcases, list_maxfat] ).T,
columns=['Country_Region', 'Province_State', 'Start_Pandemic', 'ConfirmedCases', 'Fatalities'])
 dfstartpand['Start_Pandemic'] = dfstartpand['Start_Pandemic'].dt.date
train = train.append(test[test['Date']>'2020-03-31'] )<feature_engineering>
dfstartpand_ordered = dfstartpand.sort_values(by=['Start_Pandemic', 'ConfirmedCases', 'Fatalities'], ascending=[True, False, False])
 country_state_ordered = list(zip(dfstartpand_ordered['Country_Region'].tolist() , dfstartpand_ordered['Province_State']))
datetrain = dftrain['Date'].unique()
datetest = dftest['Date'].unique()
train['Date'] = pd.to_datetime(train['Date'], format='%Y-%m-%d')
 train['day_dist'] = train['Date']-train['Date'].min()
train['day_dist'] = train['day_dist'].dt.days
print(train['Date'].max())
 print(test['Date'].min())
 print(test['Date'].max())
 cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols
for cat_col in cat_cols:
train[cat_col].fillna('no_value', inplace = True)
 train['place'] = train['Province_State']+'_'+train['Country_Region'] 
<categorify>
dftest['ConfirmedCases_logreg'] = 0.0 ; dftrain['ConfirmedCases_logreg'] = 0.0
dftest['Fatalities_logreg'] = 0.0 ; dftrain['Fatalities_logreg'] = 0.0
p0 = 1
for country, state in country_state_ordered:
masktrain =(dftrain['Country_Region'] == country)&(dftrain['Province_State'] == state)
 masktrain2 =(dftrain['Country_Region'] == country)&(dftrain['Province_State'] == state)& \ 
(dftrain['Date'] <= '2020-04-07')&(dftrain['Date'] >= starttest)
 masktest =(dftest['Country_Region'] == country)&(dftest['Province_State'] == state)
 masktest2 =(dftest['Country_Region'] == country)&(dftest['Province_State'] == state)& \ 
(dftest['Date'] <= '2020-04-07')
 df2plot = dftrain[masktrain].set_index('Date')
 X = np.arange(len(df2plot))
X_test =(np.timedelta64(datetest[0]-datetrain[0], 'D')).astype(float)+np.arange(0,len(datetest))
try:
y = df2plot['ConfirmedCases']
p0_cases = [1/(len(X)/2.) , X[-1], y.max() ]
popt, pcov = curve_fit(logistic_curve, X, y, p0=p0_cases,bounds=([0,0,0],np.inf), maxfev=1000)
 k_cases, x_0_cases, ymax_cases = popt
cases_train_fc = pd.Series(logistic_curve(X, k_cases, x_0_cases, ymax_cases),index=df2plot.index)
 cases_test_fc = pd.Series(logistic_curve(X_test, k_cases, x_0_cases, ymax_cases),index=datetest)
 dftest.loc[masktest,'ConfirmedCases_logreg'] = cases_test_fc.tolist()
dftrain.loc[masktrain,'ConfirmedCases_logreg'] = cases_train_fc.tolist()
except:
print(country+' '+state+' Unable to fit the confirmed cases')
 dftest.loc[masktest,'ConfirmedCases_logreg'] = dftrain.loc[masktrain,'ConfirmedCases'].iloc[-1]
dftrain.loc[masktrain,'ConfirmedCases_logreg'] = dftrain.loc[masktrain,'ConfirmedCases']
try:
y = df2plot['Fatalities']
p0_deaths = [1/(len(X)/2.) , X[-1], y.max() ]
popt, pcov = curve_fit(logistic_curve, X, y, p0=p0_deaths,bounds=([0,0,0],np.inf), maxfev=1000)
 k_deaths, x_0_deaths, ymax_deaths = popt
deaths_train_fc = pd.Series(logistic_curve(X, k_deaths, x_0_deaths, ymax_deaths),index=datetrain)
 deaths_test_fc = pd.Series(logistic_curve(X_test, k_deaths, x_0_deaths, ymax_deaths),index=datetest)
 dftest.loc[masktest,'Fatalities_logreg'] = deaths_test_fc.tolist()
dftrain.loc[masktrain,'Fatalities_logreg'] = deaths_train_fc.tolist()
except:
print(country+' '+state+' Unable to fit the fatalities')
 dftest.loc[masktest,'Fatalities_logreg'] = dftrain.loc[masktrain,'Fatalities'].iloc[-1]
dftrain.loc[masktrain,'Fatalities_logreg'] = dftrain.loc[masktrain,'Fatalities']
dftest.loc[masktest2,'ConfirmedCases_logreg'] = dftrain.loc[masktrain2,'ConfirmedCases'].tolist()
dftest.loc[masktest2,'Fatalities_logreg'] = dftrain.loc[masktrain2,'Fatalities'].tolist()
cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols
for cat_col in ['place']:
le = preprocessing.LabelEncoder()
le.fit(train[cat_col])
 train[cat_col]=le.transform(train[cat_col])
 train.keys()
drop_cols = ['Id', 'ConfirmedCases','Date', 'ForecastId','Fatalities','day_dist', 'Province_State', 'Country_Region']<filter>
dfsubm['ConfirmedCases'] = dftest['ConfirmedCases_logreg']
dfsubm['Fatalities'] = dftest['Fatalities_logreg']
val = train[(train['Date']>='2020-03-19')&(train['Id'].isnull() ==False)]
val<prepare_x_and_y>
dfsubm.to_csv('submission.csv', index=False )