Kaggle competition: COVID19 Global Forecasting (Week 3)
df = pd.DataFrame(columns = ['ForecastId','ConfirmedCases','Fatalities'])
 df_hr = pd.DataFrame(columns = ['ForecastId', 'Country_Region', 'Province_State', 'Days', 'ConfirmedCases','Fatalities','Date'])
<define_variables>
data= pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/test.csv" )
public_start_date = '2020-03-19'
public_end_date = '2020-04-01'
count = 0
for index, start_date in zip(min_date_sorted.index, min_date_sorted):
print(list(min_date_sorted.index ).index(index), index, start_date)
 record = df_train[(df_train['Country_Region'] == index[0])&(df_train['Province_State'] == index[1])]
if len(record[record['ConfirmedCases'] > 0])== 0:
pass
else:
record = record[record['ConfirmedCases'] > 0]
base_date_object = datetime.strptime(start_date, "%Y-%m-%d" ).date()
public_start_date_object = datetime.strptime(public_start_date, "%Y-%m-%d" ).date()
public_end_date_object = datetime.strptime(public_end_date, "%Y-%m-%d" ).date()
record['days'] = [(datetime.strptime(date, "%Y-%m-%d" ).date() - base_date_object ).days + 1 for date in record['Date']]
public_start_day =(public_start_date_object - base_date_object ).days + 1
public_end_day =(public_end_date_object - base_date_object ).days + 1
if len(record[record['days'] < public_start_day])> 0:
record = record[record['days'] < public_start_day]
record2 = record
if len(record[record['Fatalities'] > 0])!= 0:
record2 = record[record['Fatalities'] > 0]
x = record['days'].values
x2 = record2['days'].values
y1 = record['ConfirmedCases'].values
y2 = record2['Fatalities'].values
b = -1
bad_index = 0
mod_count = 0
for a in y1:
if a < b:
y1[bad_index] = b
mod_count = mod_count + 1
else:
b = a
bad_index = bad_index + 1
if mod_count > 0:
print("*****************")
 print(list(min_date_sorted.index ).index(index), index)
 print(mod_count)
 print(y1)
 print("*****************")
 b = -1
bad_index = 0
mod_count = 0
for a in y2:
if a < b:
y2[bad_index] = b
mod_count = mod_count + 1
else:
b = a
bad_index = bad_index + 1
if mod_count > 0:
print("*****************")
 print(list(min_date_sorted.index ).index(index), index)
 print(mod_count)
 print(y2)
 print("*****************")
 if len(y1)> 0:
y_max_ = y1[-1]
y1_prime = np.diff(y1)
 y1_prime2 = np.diff(y1_prime)
 if len(y1)>0 and len(y1_prime)> 0 and len(y1_prime2)> 0:
max_slope_index = len(y1_prime)- 1 - list(y1_prime)[::-1].index(max(y1_prime))
max_slope_range =(max_slope_index+1)/len(y1_prime)
 y_max_ = y1[-1]
if max_slope_range < 0.75:
if y1_prime[max_slope_index] > 0 and max_slope_range < 0.75 and(((y1_prime[max_slope_index] - max(y1_prime[-2:])) /y1_prime[max_slope_index])< 0.5):
y_max_ = y1[-1]
pass
else:
y_max_ = y1[max_slope_index + 1]
pass
else:
y_max_ = y1[-1]
else:
y_max_ = 0
stage0 = False
stage1 = False
stage2 = False
stage3 = False
count1 = 0
count2 = 0
for start in range(len(y1_prime)-3):
if sum(y1_prime[start:start+3])<=12:
count1 = count1 + 1
count2 = 0
else:
count2 = count2 + 1
count1 = 0
if not stage0 and count2 == 0 and count1 > 2:
stage0 = True
count1 = 0
if not stage1 and count1 == 0 and count2 > 5:
stage0 = True
stage1 = True
count2 = 0
if stage1 and count2 == 0 and count1 > 3:
stage2 = True
count1 = 0
if stage2 and count1 == 0 and count2 > 2:
stage3 = True
count2 = 0
if stage3:
param, param_cov = curve_fit(custom, np.array(x), np.array(y1), maxfev = 100000, bounds=([1, 0, 1, 1, 30, 1, 1], [2, 60, 8, 200, 90, 8, 1400]))
y1_pred = custom(x, param[0], param[1], param[2], param[3], param[4], param[5], param[6])
 elif index[0] == 'Korea, South':
param, param_cov = curve_fit(test_linear, np.array(x), np.array(y1), [y_max_, 0.5,(x[-1] - x[0])/2 + x[0], 50, 45, 0], maxfev = 100000, bounds=([y_max_/2, 0.1, 0, 1, 30, -100],[y_max_*5 + 1500, 1, 150, 100, 100, 1000]))
y1_pred = test_linear(x, param[0], param[1], param[2], param[3], param[4], param[5])
 elif index[0] in ['US', 'Spain', 'Germany', 'France', 'Iran', 'United Kingdom']:
param, param_cov = curve_fit(test, np.array(x), np.array(y1), [y_max_*7, 0.5,(x[-1] - x[0])/2 + x[0]], maxfev = 100000, bounds=([y_max_*5, 0.1, 0],[y_max_*10 + 1500, 1, 150]))
y1_pred = test(x, param[0], param[1], param[2])
 elif index[0] == 'China':
param, param_cov = curve_fit(test, np.array(x), np.array(y1), [y_max_, 0.5,(x[-1] - x[0])/2 + x[0]], maxfev = 100000, bounds=([y_max_/2, 0.1, 0],[y_max_*5 + 1500, 1, 150]))
y1_pred = test(x, param[0], param[1], param[2])
 else:
param, param_cov = curve_fit(test, np.array(x), np.array(y1), [y_max_*5, 0.5,(x[-1] - x[0])/2 + x[0]], maxfev = 100000, bounds=([y_max_*4, 0.1, 0],[y_max_*8 + 1500, 1, 150]))
y1_pred = test(x, param[0], param[1], param[2])
 param2, param_cov2 = curve_fit(test, np.array(x2), np.array(y2), [y2[-1]/2, 0.5,(x2[-1] - x2[0])/2 + x2[0] - 3], maxfev = 100000, bounds=([y2[-1]/2, 0.1, 0],[y2[-1]*5 + 1, 0.8, 150]))
y2_pred = test(x2,param2[0], param2[1], param2[2])
 group = test_groups.get_group(index)
 group['days'] = [(datetime.strptime(date, "%Y-%m-%d" ).date() - base_date_object ).days + 1 for date in group['Date'].values]
group = group[group['days'] <= public_end_day]
ids = group['ForecastId'].values
days = [(datetime.strptime(date, "%Y-%m-%d" ).date() - base_date_object ).days + 1 for date in group['Date'].values]
prev_days = range(public_start_day - 6, public_start_day - 1, 1)
 if stage3:
test_y1_pred_raw = custom(days, param[0], param[1], param[2], param[3], param[4], param[5], param[6])
 prev_y1_pred_raw = custom(prev_days, param[0], param[1], param[2], param[3], param[4], param[5], param[6])
 elif index[0] == 'Korea, South':
test_y1_pred_raw = test_linear(days, param[0], param[1], param[2], param[3], param[4], param[5])
 prev_y1_pred_raw = test_linear(prev_days, param[0], param[1], param[2], param[3], param[4], param[5])
 elif index[0] in ['US', 'Spain', 'Germany', 'France', 'Iran', 'United Kingdom']:
test_y1_pred_raw = test(days, param[0], param[1], param[2])
 prev_y1_pred_raw = test(prev_days, param[0], param[1], param[2])
 else:
test_y1_pred_raw = test(days, param[0], param[1], param[2])
 prev_y1_pred_raw = test(prev_days, param[0], param[1], param[2])
 test_y2_pred_raw = test(days, param2[0], param2[1], param2[2])
 prev_y2_pred_raw = test(prev_days, param2[0], param2[1], param2[2])
 day_index_pred = 0
diff1_list = []
diff2_list = []
for day in prev_days:
if day in x:
day_index = np.where(x == day)
 diff1 = y1[day_index] - prev_y1_pred_raw[day_index_pred]
diff1_list.append(diff1)
 if day in x2:
day_index = np.where(x2 == day)
 diff2 = y2[day_index] - prev_y2_pred_raw[day_index_pred]
diff2_list.append(diff2)
 day_index_pred = day_index_pred + 1
if len(diff1_list)> 0:
diff1_mean = np.max(diff1_list)
 else:
diff1_mean = 0
if len(diff2_list)> 0:
diff2_mean = np.max(diff2_list)
 else:
diff2_mean = 0
if np.isnan(diff1_mean):
pass
else:
test_y1_pred_raw = list(np.array(test_y1_pred_raw)+ diff1_mean)
 if np.isnan(diff2_mean):
pass
else:
test_y2_pred_raw = list(np.array(test_y2_pred_raw)+ diff2_mean)
 test_y1_pred = test_y1_pred_raw
test_y2_pred = test_y2_pred_raw
ratio = 0
if y2[-1] > 0:
ratio = y1[-1]/y2[-1]
else:
ratio = y1[-1]
train_day_index = days.index(public_start_day)- 1
if(y1[-1] < 100 and y2[-1] < 4 and ratio < avg):
for pred_index in range(len(test_y2_pred)) :
if pred_index > train_day_index:
if test_y2_pred[pred_index] < test_y1_pred[pred_index]/avg:
test_y2_pred[pred_index] = test_y1_pred[pred_index]/avg
else:
for pred_index in range(len(test_y2_pred)) :
if pred_index > train_day_index:
if test_y2_pred[pred_index] < test_y1_pred[pred_index]/ratio:
test_y2_pred[pred_index] = test_y1_pred[pred_index]/ratio
test_y1_pred = [int(n)for n in test_y1_pred]
test_y2_pred = [int(m)for m in test_y2_pred]
local_df_hr = pd.DataFrame(ids, columns=['ForecastId'])
 print()
local_df_hr.insert(1, 'Country_Region', [index[0]]*len(days))
local_df_hr.insert(2, 'Province_State', [index[1]]*len(days))
local_df_hr.insert(3, 'Days', days)
 local_df_hr.insert(4, 'ConfirmedCases', test_y1_pred)
 local_df_hr.insert(5, 'Fatalities', test_y2_pred)
 local_df_hr.insert(6, 'Date', group['Date'].values)
 local_df = pd.DataFrame(ids, columns=['ForecastId'])
 local_df.insert(1, 'ConfirmedCases', test_y1_pred)
 local_df.insert(2, 'Fatalities', test_y2_pred)
 df = df.append(local_df)
 df_hr = df_hr.append(local_df_hr)
 count = count + 1 
<define_variables>
data['Province_State']=data['Province_State'].fillna('')
 test['Province_State']=test['Province_State'].fillna('' )
private_start_date = '2020-04-02'
private_end_date = '2020-04-30'
count = 0
for index, start_date in zip(min_date_sorted.index, min_date_sorted):
print(list(min_date_sorted.index ).index(index), index, start_date)
 record = df_train[(df_train['Country_Region'] == index[0])&(df_train['Province_State'] == index[1])]
if len(record[record['ConfirmedCases'] > 0])== 0:
pass
else:
record = record[record['ConfirmedCases'] > 0]
base_date_object = datetime.strptime(start_date, "%Y-%m-%d" ).date()
private_start_date_object = datetime.strptime(private_start_date, "%Y-%m-%d" ).date()
private_end_date_object = datetime.strptime(private_end_date, "%Y-%m-%d" ).date()
record['days'] = [(datetime.strptime(date, "%Y-%m-%d" ).date() - base_date_object ).days + 1 for date in record['Date']]
private_start_day =(private_start_date_object - base_date_object ).days + 1
private_end_day =(private_end_date_object - base_date_object ).days + 1
if len(record[record['days'] < private_start_day])> 0:
record = record[record['days'] < private_start_day]
record2 = record
if len(record[record['Fatalities'] > 0])!= 0:
record2 = record[record['Fatalities'] > 0]
x = record['days'].values
x2 = record2['days'].values
y1 = record['ConfirmedCases'].values
y2 = record2['Fatalities'].values
b = -1
bad_index = 0
mod_count = 0
for a in y1:
if a < b:
y1[bad_index] = b
mod_count = mod_count + 1
else:
b = a
bad_index = bad_index + 1
if mod_count > 0:
print("*****************")
 print(list(min_date_sorted.index ).index(index), index)
 print(mod_count)
 print(y1)
 print("*****************")
 b = -1
bad_index = 0
mod_count = 0
for a in y2:
if a < b:
y2[bad_index] = b
mod_count = mod_count + 1
else:
b = a
bad_index = bad_index + 1
if mod_count > 0:
print("*****************")
 print(list(min_date_sorted.index ).index(index), index)
 print(mod_count)
 print(y2)
 print("*****************")
 y_max_ = y1[-1]
y1_prime = np.diff(y1)
 y1_prime2 = np.diff(y1_prime)
 if len(y1)>0 and len(y1_prime)> 0 and len(y1_prime2)> 0:
max_slope_index = len(y1_prime)- 1 - list(y1_prime)[::-1].index(max(y1_prime))
max_slope_range =(max_slope_index+1)/len(y1_prime)
 y_max_ = y1[-1]
if max_slope_range < 0.75:
if y1_prime[max_slope_index] > 0 and max_slope_range < 0.75 and(((y1_prime[max_slope_index] - max(y1_prime[-2:])) /y1_prime[max_slope_index])< 0.5):
y_max_ = y1[-1]
pass
else:
y_max_ = y1[max_slope_index + 1]
pass
else:
y_max_ = y1[-1]
stage0 = False
stage1 = False
stage2 = False
stage3 = False
count1 = 0
count2 = 0
for start in range(len(y1_prime)-3):
if sum(y1_prime[start:start+3])<=12:
count1 = count1 + 1
count2 = 0
else:
count2 = count2 + 1
count1 = 0
if not stage0 and count2 == 0 and count1 > 2:
stage0 = True
count1 = 0
if not stage1 and count1 == 0 and count2 > 5:
stage0 = True
stage1 = True
count2 = 0
if stage1 and count2 == 0 and count1 > 3:
stage2 = True
count1 = 0
if stage2 and count1 == 0 and count2 > 2:
stage3 = True
count2 = 0
if stage3:
param, param_cov = curve_fit(custom, np.array(x), np.array(y1), maxfev = 100000, bounds=([1, 0, 1, 1, 30, 1, 1], [2, 60, 8, 200, 90, 8, 1400]))
y1_pred = custom(x, param[0], param[1], param[2], param[3], param[4], param[5], param[6])
 elif index[0] == 'Korea, South':
param, param_cov = curve_fit(test_linear, np.array(x), np.array(y1), [y_max_, 0.5,(x[-1] - x[0])/2 + x[0], 50, 45, 0], maxfev = 100000, bounds=([y_max_/2, 0.1, 0, 1, 30, -100],[y_max_*5 + 1500, 1, 150, 100, 100, 1000]))
y1_pred = test_linear(x, param[0], param[1], param[2], param[3], param[4], param[5])
 elif index[0] in ['US', 'Spain', 'Germany', 'France', 'Iran', 'United Kingdom']:
param, param_cov = curve_fit(test, np.array(x), np.array(y1), [y_max_*6, 0.5,(x[-1] - x[0])/2 + x[0]], maxfev = 100000, bounds=([y_max_*5, 0.1, 0],[y_max_*10 + 1500, 1, 150]))
y1_pred = test(x, param[0], param[1], param[2])
 elif index[0] == 'China':
param, param_cov = curve_fit(test, np.array(x), np.array(y1), [y_max_, 0.5,(x[-1] - x[0])/2 + x[0]], maxfev = 100000, bounds=([y_max_/2, 0.1, 0],[y_max_*5 + 1500, 1, 150]))
y1_pred = test(x, param[0], param[1], param[2])
 elif index[0] in ['Italy', 'Switzerland']:
param, param_cov = curve_fit(test, np.array(x), np.array(y1), [y_max_*3, 0.5,(x[-1] - x[0])/2 + x[0]], maxfev = 100000, bounds=([y_max_*2, 0.1, 0],[y_max_*5 + 1500, 1, 150]))
y1_pred = test(x, param[0], param[1], param[2])
 else:
param, param_cov = curve_fit(test, np.array(x), np.array(y1), [y_max_*4, 0.5,(x[-1] - x[0])/2 + x[0]], maxfev = 100000, bounds=([y_max_*3, 0.1, 0],[y_max_*8 + 1500, 1, 150]))
y1_pred = test(x, param[0], param[1], param[2])
 param2, param_cov2 = curve_fit(test, np.array(x2), np.array(y2), [y2[-1]/2, 0.5,(x2[-1] - x2[0])/2 + x2[0] - 3], maxfev = 100000, bounds=([y2[-1]/2, 0.1, 0],[y2[-1]*5 + 1, 0.8, 150]))
y2_pred = test(x2,param2[0], param2[1], param2[2])
 group = test_groups.get_group(index)
 group['days'] = [(datetime.strptime(date, "%Y-%m-%d" ).date() - base_date_object ).days + 1 for date in group['Date'].values]
group = group[group['days'] >= private_start_day]
ids = group['ForecastId'].values
days = [(datetime.strptime(date, "%Y-%m-%d" ).date() - base_date_object ).days + 1 for date in group['Date'].values]
prev_days = range(private_start_day - 6, private_start_day - 1, 1)
 if stage3:
test_y1_pred_raw = custom(days, param[0], param[1], param[2], param[3], param[4], param[5], param[6])
 prev_y1_pred_raw = custom(prev_days, param[0], param[1], param[2], param[3], param[4], param[5], param[6])
 elif index[0] == 'Korea, South':
test_y1_pred_raw = test_linear(days, param[0], param[1], param[2], param[3], param[4], param[5])
 prev_y1_pred_raw = test_linear(prev_days, param[0], param[1], param[2], param[3], param[4], param[5])
 elif index[0] in ['US', 'Spain', 'Germany', 'France', 'Iran', 'United Kingdom']:
test_y1_pred_raw = test(days, param[0], param[1], param[2])
 prev_y1_pred_raw = test(prev_days, param[0], param[1], param[2])
 else:
test_y1_pred_raw = test(days, param[0], param[1], param[2])
 prev_y1_pred_raw = test(prev_days, param[0], param[1], param[2])
 test_y2_pred_raw = test(days, param2[0], param2[1], param2[2])
 prev_y2_pred_raw = test(prev_days, param2[0], param2[1], param2[2])
 day_index_pred = 0
diff1_list = []
diff2_list = []
for day in prev_days:
if day in x:
day_index = np.where(x == day)
 diff1 = y1[day_index] - prev_y1_pred_raw[day_index_pred]
diff1_list.append(diff1)
 if day in x2:
day_index = np.where(x2 == day)
 diff2 = y2[day_index] - prev_y2_pred_raw[day_index_pred]
diff2_list.append(diff2)
 day_index_pred = day_index_pred + 1
if len(diff1_list)> 0:
diff1_mean = np.max(diff1_list)
 else:
diff1_mean = 0
if len(diff2_list)> 0:
diff2_mean = np.max(diff2_list)
 else:
diff2_mean = 0
if np.isnan(diff1_mean):
pass
else:
test_y1_pred_raw = list(np.array(test_y1_pred_raw)+ diff1_mean)
 if np.isnan(diff2_mean):
pass
else:
test_y2_pred_raw = list(np.array(test_y2_pred_raw)+ diff2_mean)
 test_y1_pred = test_y1_pred_raw
test_y2_pred = test_y2_pred_raw
ratio = 0
if y2[-1] > 0:
ratio = y1[-1]/y2[-1]
else:
ratio = y1[-1]
train_day_index = days.index(private_start_day)- 1
if(y1[-1] < 100 and y2[-1] < 4 and ratio < avg):
for pred_index in range(len(test_y2_pred)) :
if pred_index > train_day_index:
if test_y2_pred[pred_index] < test_y1_pred[pred_index]/avg:
test_y2_pred[pred_index] = test_y1_pred[pred_index]/avg
else:
for pred_index in range(len(test_y2_pred)) :
if pred_index > train_day_index:
if test_y2_pred[pred_index] < test_y1_pred[pred_index]/ratio:
test_y2_pred[pred_index] = test_y1_pred[pred_index]/ratio
test_y1_pred = [int(n)for n in test_y1_pred]
test_y2_pred = [int(m)for m in test_y2_pred]
local_df_hr = pd.DataFrame(ids, columns=['ForecastId'])
 local_df_hr.insert(1, 'Country_Region', [index[0]]*len(days))
local_df_hr.insert(2, 'Province_State', [index[1]]*len(days))
local_df_hr.insert(3, 'Days', days)
 local_df_hr.insert(4, 'ConfirmedCases', test_y1_pred)
 local_df_hr.insert(5, 'Fatalities', test_y2_pred)
 local_df_hr.insert(6, 'Date', group['Date'].values)
 local_df = pd.DataFrame(ids, columns=['ForecastId'])
 local_df.insert(1, 'ConfirmedCases', test_y1_pred)
 local_df.insert(2, 'Fatalities', test_y2_pred)
 df = df.append(local_df)
 df_hr = df_hr.append(local_df_hr)
 count = count + 1 
<sort_values>
import matplotlib.pyplot as plt
df = df.sort_values(by=['ForecastId'], ascending=True)
 df_hr = df_hr.sort_values(by=['ForecastId'], ascending=True )<save_to_csv>
datetime_str = '01/22/20 00:00:00'
datetime_object = datetime.strptime(datetime_str, '%m/%d/%y %H:%M:%S')
 data['days']=pd.to_datetime(data['Date'] ).sub(datetime_object)/np.timedelta64(1, 'D')
 test['days']=pd.to_datetime(test['Date'] ).sub(datetime_object)/np.timedelta64(1, 'D' )
df.to_csv('submission.csv', index=False)
 df_hr.to_csv('hr_submission.csv', index=False )<import_modules>
data.loc[(data['Province_State']=='')&(data['Country_Region']=='India'),:].sort_values(by="Date" )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt<define_variables>
data.to_csv("train_1.csv")
 test.to_csv("test_1.csv")
EMPTY_VAL = "EMPTY_VAL"
NAN_VAL = "NaN"
def get_state(state, country):
if state == EMPTY_VAL: return country
if state == NAN_VAL: return country
return state<compute_test_metric>
from statsmodels.tsa.arima_model import ARIMA
def calc_score(y_true, y_pred):
y_true[y_true<0] = 0
score = metrics.mean_squared_error(np.log(y_true.clip(0, 1e10)+1), np.log(y_pred[:]+1)) **0.5
return score<load_from_csv>
data['Date']=pd.to_datetime(data['Date'])
 test['Date']=pd.to_datetime(test['Date'] )
PATH_WEEK2='/kaggle/input/covid19-global-forecasting-week-2'
df_train = pd.read_csv(f'{PATH_WEEK2}/train.csv')
 df_test = pd.read_csv(f'{PATH_WEEK2}/test.csv')
 df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True)
 y1_Train = df_train.iloc[:, -2]
y2_Train = df_train.iloc[:, -1] 
<data_type_conversions>
pd.DataFrame(data.loc[data['Country_Region']=='Afghanistan',['ConfirmedCases']] ).reset_index(drop=True )
X_Train = df_train.copy()
X_Train['State'].fillna(EMPTY_VAL, inplace=True)
 X_Train['State'] = X_Train.loc[:, ['State', 'Country']].apply(lambda x : get_state(x['State'], x['Country']), axis=1)
 X_Train.loc[:, 'Date'] = X_Train.Date.dt.strftime("%m%d")
 X_Train["Date"] = X_Train["Date"].astype(int)
 X_Train.head()<data_type_conversions>
data.isna().sum(axis=0 )
X_Pred = df_test.copy()
X_Pred['State'].fillna(EMPTY_VAL, inplace=True)
 X_Pred['State'] = X_Pred.loc[:, ['State', 'Country']].apply(lambda x : get_state(x['State'], x['Country']), axis=1)
 X_Pred.loc[:, 'Date'] = X_Pred.Date.dt.strftime("%m%d")
 X_Pred["Date"] = X_Pred["Date"].astype(int)
 X_Pred.head()<categorify>
data['ConfirmedCases'][data['Country_Region']==''][51:]
le = preprocessing.LabelEncoder()
X_Train.Country = le.fit_transform(X_Train.Country)
 X_Train['State'] = le.fit_transform(X_Train['State'])
 X_Train.head()<categorify>
data['ConfirmedCases'][data['Country_Region']=='India'].value_counts()
X_Pred.Country = le.fit_transform(X_Pred.Country)
 X_Pred['State'] = le.fit_transform(X_Pred['State'])
 X_Pred.head()<create_dataframe>
pd.DataFrame(data.loc[data['Country_Region']=='India',['ConfirmedCases']] )
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
n_estimators = 5000
n_poly_degree = 5
countries = X_Train.Country.unique()
df_out_xgb = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 df_out_lgb = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 df_out_ply = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_Train.loc[X_Train.Country == country, :].State.unique()
for state in states:
X_Train_CS = X_Train.loc[(X_Train.Country == country)&(X_Train.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y_Train_CS_Cases = X_Train_CS.loc[:, 'ConfirmedCases']
y_Train_CS_Fatal = X_Train_CS.loc[:, 'Fatalities']
X_Train_CS = X_Train_CS.loc[:, ['State', 'Country', 'Date']]
X_Train_CS.Country = le.fit_transform(X_Train_CS.Country)
 X_Train_CS['State'] = le.fit_transform(X_Train_CS['State'])
 X_Pred_CS = X_Pred.loc[(X_Pred.Country == country)&(X_Pred.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_Pred_CS_Id = X_Pred_CS.loc[:, 'ForecastId']
X_Pred_CS = X_Pred_CS.loc[:, ['State', 'Country', 'Date']]
X_Pred_CS.Country = le.fit_transform(X_Pred_CS.Country)
 X_Pred_CS['State'] = le.fit_transform(X_Pred_CS['State'])
 model_xgb_cases = XGBRegressor(n_estimators = n_estimators)
 model_xgb_cases.fit(X_Train_CS, y_Train_CS_Cases)
 y_xgb_cases_pred = model_xgb_cases.predict(X_Pred_CS)
 model_xgb_fatal = XGBRegressor(n_estimators = n_estimators)
 model_xgb_fatal.fit(X_Train_CS, y_Train_CS_Fatal)
 y_xgb_fatal_pred = model_xgb_fatal.predict(X_Pred_CS)
 model_lgb_cases = XGBRegressor(n_estimators = n_estimators)
 model_lgb_cases.fit(X_Train_CS, y_Train_CS_Cases)
 y_lgb_cases_pred = model_lgb_cases.predict(X_Pred_CS)
 model_lgb_fatal = XGBRegressor(n_estimators = n_estimators)
 model_lgb_fatal.fit(X_Train_CS, y_Train_CS_Fatal)
 y_lgb_fatal_pred = model_lgb_fatal.predict(X_Pred_CS)
 pl_cases_pred = np.poly1d(np.polyfit(X_Train_CS.Date, y_Train_CS_Cases, n_poly_degree))
y_ply_cases_pred = pl_cases_pred(X_Pred_CS.Date)
 pl_fatal_pred = np.poly1d(np.polyfit(X_Train_CS.Date, y_Train_CS_Fatal, n_poly_degree))
y_ply_fatal_pred = pl_fatal_pred(X_Pred_CS.Date)
 df_xgb = pd.DataFrame({'ForecastId': X_Pred_CS_Id, 'ConfirmedCases': y_xgb_cases_pred, 'Fatalities': y_xgb_fatal_pred})
 df_lgb = pd.DataFrame({'ForecastId': X_Pred_CS_Id, 'ConfirmedCases': y_lgb_cases_pred, 'Fatalities': y_lgb_fatal_pred})
 df_ply = pd.DataFrame({'ForecastId': X_Pred_CS_Id, 'ConfirmedCases': y_ply_cases_pred, 'Fatalities': y_ply_fatal_pred})
 df_out_xgb = pd.concat([df_out_xgb, df_xgb], axis=0)
 df_out_lgb = pd.concat([df_out_lgb, df_lgb], axis=0)
 df_out_ply = pd.concat([df_out_ply, df_ply], axis=0)
<data_type_conversions>
datetime_str = '03/22/20 00:00:00'
datetime_object = datetime.strptime(datetime_str, '%m/%d/%y %H:%M:%S' )
df_out_xgb.ForecastId = df_out_xgb.ForecastId.astype('int')
 df_out_lgb.ForecastId = df_out_lgb.ForecastId.astype('int')
 df_out_ply.ForecastId = df_out_ply.ForecastId.astype('int' )<create_dataframe>
from datetime import timedelta
df_out = df_out_ply.copy()<feature_engineering>
import math
df_out['ConfirmedCases'] =(1/4)*(df_out_xgb['ConfirmedCases'] + df_out_lgb['ConfirmedCases'])+(1/2)* df_out_ply['ConfirmedCases']
df_out['Fatalities'] =(1/4)*(df_out_xgb['Fatalities'] + df_out_lgb['Fatalities'])+(1/2)* df_out_ply['Fatalities']<data_type_conversions>
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
df_out['ConfirmedCases'] = df_out['ConfirmedCases'].round().astype(int)
 df_out['Fatalities'] = df_out['Fatalities'].round().astype(int )<save_to_csv>
def evaluate_arima_model(X,forecast_days, arima_order):
X=[x for x in X]
train_size = int(len(X)* 0.8)
 train, test1 = X[0:train_size], X[train_size:]
history=train
model = ARIMA(history, order=arima_order)
 model_fit = model.fit(disp=0)
 predictions = list()
predictions =model_fit.forecast(steps=len(test1)) [0]
model = ARIMA(X, order=arima_order)
 model_fit = model.fit(disp=0)
 if np.isnan(model_fit.forecast(steps=forecast_days)[0] ).sum() >0:
return float('inf')
 error = rmsle(test1, predictions)
 return error
df_out.to_csv('submission.csv', index=False )<import_modules>
def evaluate_models(dataset,forcast_days, p_values, d_values, q_values):
best_score, best_cfg = float("inf"),(0,0,0)
 for p in p_values:
for d in d_values:
for q in q_values:
order =(p,d,q)
 try:
mse = evaluate_arima_model(dataset,forcast_days, order)
 if mse < best_score:
best_score, best_cfg = mse, order
except:
continue
print('Best ARIMA%s MSE=%.3f' %(best_cfg, best_score))
return best_cfg, best_score
import numpy as np
import pandas as pd
import lightgbm as lgb
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit<load_from_csv>
warnings.filterwarnings('ignore' )
train = pd.read_csv('.. /input/covid19-global-forecasting-week-2/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-2/test.csv')
 world_population = pd.read_csv('.. /input/population-by-country-2020/population_by_country_2020.csv' )<merge>
test['ConfirmedCases']=0
test['Fatalities']=0
world_population = world_population[['Country(or dependency)', 'Population(2020)']]
world_population.columns = ['Country(or dependency)', 'Population']
world_population.loc[world_population['Country(or dependency)']=='United States', 'Country(or dependency)'] = 'US'
train = train.merge(world_population, left_on='Country_Region', right_on='Country(or dependency)', how='left')
 test = test.merge(world_population, left_on='Country_Region', right_on='Country(or dependency)', how='left' )<feature_engineering>
sliced_data=data.loc[(data['Province_State']=='')&(data['Country_Region']=='India'),:]
train['State_Country'] = [s + '_' + c if s == s else c for s,c in train[['Province_State', 'Country_Region']].values ]
test['State_Country'] = [s + '_' + c if s == s else c for s,c in test[['Province_State', 'Country_Region']].values ]<feature_engineering>
from pandas import read_csv
from pandas import datetime
from matplotlib import pyplot
from statsmodels.tsa.arima_model import ARIMA
from sklearn.metrics import mean_squared_error
from math import sqrt
from time import time
from sklearn.metrics import mean_squared_error
train.loc[(train['Date']=='2020-03-24')&(train['State_Country']=='France'),'ConfirmedCases'] = 22654
train.loc[(train['Date']=='2020-03-24')&(train['State_Country']=='France'),'Fatalities'] = 1000<groupby>
country='India'
state=''
sliced_data=data.loc[(data['Province_State']==state)&(data['Country_Region']==country),:]
test_sliced=test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:]
print(sliced_data)
 sliced_data=sliced_data.drop_duplicates()
sliced_data=sliced_data.reset_index(drop=True)
 sliced_data=sliced_data.sort_values(by='Date')
 if sliced_data.loc[sliced_data['ConfirmedCases']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['ConfirmedCases']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['ConfirmedCases'].to_list()
print('history')
 print(history)
 if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(10),range(7),range(7))
preds=[]
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions=pd.DataFrame()
predictions['Date']=dates
predictions['ConfirmedCases']=preds
test_sliced=test_sliced.merge(sliced_data[['Date','ConfirmedCases']], on='Date',how='left')
 test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
test_sliced=test_sliced.merge(predictions, on='Date',how='left')
 test_sliced['ConfirmedCases_x'][test_sliced['ConfirmedCases_x'].isna() ]=test_sliced['ConfirmedCases_y'][test_sliced['ConfirmedCases_x'].isna() ]
test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_x']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
sliced_data_bck=sliced_data.copy()
if sliced_data.loc[sliced_data['Fatalities']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['Fatalities']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['Fatalities'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(5),range(5),range(5))
preds=[]
model=None
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions_f=pd.DataFrame()
predictions_f['Date']=dates
predictions_f['Fatalities']=preds
test_sliced=test_sliced.merge(sliced_data_bck[['Date','Fatalities']], on='Date',how='left')
 test_sliced['Fatalities']=test_sliced['Fatalities_y']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test_sliced=test_sliced.merge(predictions_f, on='Date',how='left')
 test_sliced['Fatalities_x'][test_sliced['Fatalities_x'].isna() ]=test_sliced['Fatalities_y'][test_sliced['Fatalities_x'].isna() ]
test_sliced['Fatalities']=test_sliced['Fatalities_x']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test=test.merge(test_sliced,on='ForecastId',how='left')
 test['ConfirmedCases_x'][test['ConfirmedCases_y'].notna() ]=test['ConfirmedCases_y'][test['ConfirmedCases_y'].notna() ]
test['Fatalities_x'][test['Fatalities_y'].notna() ]=test['Fatalities_y'][test['Fatalities_y'].notna() ]
new_cols=[]
for col in test.columns:
if col[-2:]=='_y':
del test[col]
elif col[-2:]=='_x':
new_cols.append(col[:-2])
 else:
new_cols.append(col)
 test.columns=new_cols
test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:].head()
plt.plot('Date', 'ConfirmedCases', data=sliced_data, color='blue', linewidth=2)
 plt.plot('Date','ConfirmedCases',data=test_sliced,color='orange',linewidth=2)
 plt.plot('Date', 'Fatalities', data=sliced_data, color='purple', linewidth=2)
 plt.plot('Date','Fatalities',data=test_sliced,color='red',linewidth=2)
 plt.show()
for metric in ['ConfirmedCases', 'Fatalities']:
dict_values = train.groupby('State_Country')[metric].apply(np.array ).to_dict()
for country in dict_values:
if sum(np.diff(dict_values[country])< 0):
print(country, metric)
 new_val = [dict_values[country][-1]]
for val_1, val_2 in zip(dict_values[country][1:][::-1], dict_values[country][:-1][::-1]):
if val_2 <= new_val[-1]:
new_val += [val_2]
else:
new_val += [new_val[-1]]
new_val = np.array(new_val[::-1])
 train.loc[train.State_Country == country, metric] = new_val<feature_engineering>
test.loc[(test['Province_State']==state)&(test['Country_Region']==country),['Country_Region','Date','ConfirmedCases','Fatalities']]
train['max_case'] = train['State_Country'].map(train.groupby('State_Country' ).ConfirmedCases.max())
 train['pct_c'] = train.ConfirmedCases.pct_change()<predict_on_test>
test['ConfirmedCases']=0
test['Fatalities']=0
def predict_cc(data, country, len_predict, metrics, len_intersection):
country_data = data[data['State_Country']==country]
if country_data[metrics].values.max() > 1000:
start_people = 2
else:
start_people = 0
country_data = country_data.iloc[dict_case_date[country][start_people]:, :]
x_data = range(len(country_data.index))
y_data = country_data[metrics].values
if len(x_data)<= 1:
y_min = 0
y_max = 100
y_val = np.arange(len(x_data), len(x_data)+ len_predict)
 return [-1, -1, -1], log_curve(y_val, 0.3, 30, 100)
 else:
add_max =(1 + country_data.pct_c.values[-3:].mean())**(1 / 2)
 if y_data.max() > 10000:
add_min =(1 + country_data.pct_c.values[-3:].mean())**(1 / 5)
 elif y_data.max() > 1000:
add_min =(1 + country_data.pct_c.values[-3:].mean())**(1 / 3)
 else:
add_min =(1 + country_data.pct_c.values[-3:].mean())**(1 / 2)
 add_max =(1 + country_data.pct_c.values[-3:].mean())**(1)
 day_left = max(10, 50 - len(x_data))
y_min = y_data[-1] *(add_min ** day_left)
 if y_min > country_data['Population'].values[0] * 0.05:
y_min = country_data['Population'].values[0] * 0.05
y_max = y_data[-1] *(add_max ** day_left + 30)
 if y_max > country_data['Population'].values[0] * 0.15:
y_max = country_data['Population'].values[0] * 0.15
if add_min == add_max or add_max != add_max:
y_min = y_data[-1]
y_max = max(100, y_data[-1] *(1.3 ** 30))
if y_max > 100000 and y_data[-1] < 10000:
y_max = y_data[-1] * 10
y_min = y_data[-1]
popt, pcov = curve_fit(log_curve, x_data, y_data, bounds=([0.1, 10, y_min ],[0.35, 50, y_max]),
p0=[0.2,30,(y_min + y_max)/ 2], maxfev=10000)
 y_val = np.arange(len(x_data)- len_intersection, len(x_data)+ len_predict - len_intersection)
 return popt, log_curve(y_val, popt[0], popt[1], popt[2])
 def log_curve(x, k, x_0, ymax):
return ymax /(1 + np.exp(-k*(x-x_0)) )<groupby>
for country in countries:
for state in countries[country]:
print('Country : '+country,'State : '+state)
 sliced_data=data.loc[(data['Province_State']==state)&(data['Country_Region']==country),:]
test_sliced=test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:]
sliced_data=sliced_data.drop_duplicates()
sliced_data=sliced_data.reset_index(drop=True)
 sliced_data=sliced_data.sort_values(by='Date')
 if sliced_data.loc[sliced_data['ConfirmedCases']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['ConfirmedCases']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['ConfirmedCases'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(5),range(5),range(5))
preds=[]
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions=pd.DataFrame()
predictions['Date']=dates
predictions['ConfirmedCases']=preds
test_sliced=test_sliced.merge(sliced_data[['Date','ConfirmedCases']], on='Date',how='left')
 test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
test_sliced=test_sliced.merge(predictions, on='Date',how='left')
 test_sliced['ConfirmedCases_x'][test_sliced['ConfirmedCases_x'].isna() ]=test_sliced['ConfirmedCases_y'][test_sliced['ConfirmedCases_x'].isna() ]
test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_x']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
sliced_data_bck=sliced_data.copy()
if sliced_data.loc[sliced_data['Fatalities']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['Fatalities']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['Fatalities'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(5),range(5),range(5))
preds=[]
model=None
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions_f=pd.DataFrame()
predictions_f['Date']=dates
predictions_f['Fatalities']=preds
test_sliced=test_sliced.merge(sliced_data_bck[['Date','Fatalities']], on='Date',how='left')
 test_sliced['Fatalities']=test_sliced['Fatalities_y']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test_sliced=test_sliced.merge(predictions_f, on='Date',how='left')
 test_sliced['Fatalities_x'][test_sliced['Fatalities_x'].isna() ]=test_sliced['Fatalities_y'][test_sliced['Fatalities_x'].isna() ]
test_sliced['Fatalities']=test_sliced['Fatalities_x']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test=test.merge(test_sliced,on='ForecastId',how='left')
 test['ConfirmedCases_x'][test['ConfirmedCases_y'].notna() ]=test['ConfirmedCases_y'][test['ConfirmedCases_y'].notna() ]
test['Fatalities_x'][test['Fatalities_y'].notna() ]=test['Fatalities_y'][test['Fatalities_y'].notna() ]
new_cols=[]
for col in test.columns:
if col[-2:]=='_y':
del test[col]
elif col[-2:]=='_x':
new_cols.append(col[:-2])
 else:
new_cols.append(col)
 test.columns=new_cols
print(test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:].head())
 plt.plot('Date', 'ConfirmedCases', data=sliced_data, color='blue', linewidth=2)
 plt.plot('Date','ConfirmedCases',data=test_sliced,color='orange',linewidth=2)
 plt.plot('Date', 'Fatalities', data=sliced_data, color='purple', linewidth=2)
 plt.plot('Date','Fatalities',data=test_sliced,color='red',linewidth=2)
 plt.show()
metrics = 'ConfirmedCases'
dict_values = train.groupby('State_Country')[metrics].apply(np.array ).to_dict()
dict_case_date = {}
for country in dict_values:
dict_case_date[country] = []
for case in [1, 10, 100, 250, 500, 1000, 2500, 5000]:
try:
dict_case_date[country] += [np.where(dict_values[country] >= case)[0][0]]
except:
dict_case_date[country] += [-1]
dict_case_date[country] = np.array(dict_case_date[country])
 dict_predict = {}
data_train = train.copy()
data_val = test.copy()
len_predict = data_val[data_val.State_Country == country].shape[0]
len_intersection = len(set(data_train.Date.unique())& set(data_val.Date.unique()))
for country in train.State_Country.unique() :
popt, pred_values = predict_cc(data_train, country, len_predict, metrics, len_intersection)
 dict_predict[country] = pred_values<feature_engineering>
test.to_csv("test_2.csv" )
<find_best_params><EOS> 
sumb=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/submission.csv')
 output=pd.DataFrame()
output['ForecastId']=test['ForecastId'].astype(int)
 output['ConfirmedCases']=test['ConfirmedCases'].astype(int)
 output['Fatalities']=test['Fatalities'].astype(int)
 output.to_csv('submission.csv',index=False)