Kaggle competition: COVID19 Global Forecasting (Week 3)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-week-3<merge>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
from statsmodels.tsa.statespace.sarimax import SARIMAX
from statsmodels.tsa.arima_model import ARIMA
from random import random
train_new=train[["Date","ConfirmedCases","Fatalities","key","rate_ConfirmedCases","rate_Fatalities"]]
test_new=pd.merge(test,train_new, how="left", left_on=["key","Date"], right_on=["key","Date"] ).reset_index(drop=True)
 test_new<categorify>
PATH_WEEK2='/kaggle/input/covid19-global-forecasting-week-3'
df_train = pd.read_csv(f'{PATH_WEEK2}/train.csv')
 df_test = pd.read_csv(f'{PATH_WEEK2}/test.csv')
 df_train.head()
df_test.head()
df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True)
 df_train.info()
df_test.info()
y1_Train = df_train.iloc[:, -2]
y1_Train.head()
y2_Train = df_train.iloc[:, -1]
y2_Train.head()
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state
def fillin_columns(frame,key_column, original_name, training_horizon, test_horizon, unique_values, key_to_values):
keys=frame[key_column].values
original_values=frame[original_name].values.tolist()
print(len(keys), len(original_values), training_horizon ,test_horizon,len(key_to_values))
for j in range(unique_values):
current_index=(j *(training_horizon +test_horizon)) +training_horizon
current_key=keys[current_index]
values=key_to_values[current_key]
co=0
for g in range(current_index, current_index + test_horizon):
original_values[g]=values[co]
co+=1
frame[original_name]=original_values
all_days=int(test_new.shape[0]/len(unique_keys))
tr_horizon=all_days-horizon
print(all_days,tr_horizon, horizon)
 fillin_columns(test_new,"key", 'ConfirmedCases', tr_horizon, horizon, len(unique_keys), key_to_confirmed)
 fillin_columns(test_new,"key", 'Fatalities', tr_horizon, horizon, len(unique_keys), key_to_fatality)
 submission=test_new[["ForecastId","ConfirmedCases","Fatalities"]]
submission.to_csv("submission.csv", index=False)
<load_from_csv>
X_Train = df_train.copy()
X_Train['State'].fillna(EMPTY_VAL, inplace=True)
 X_Train['State'] = X_Train.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Train.loc[:, 'Date'] = X_Train.Date.dt.strftime("%m%d")
 X_Train["Date"] = X_Train["Date"].astype(int)
 X_Train.head()
X_Test = df_test.copy()
X_Test['State'].fillna(EMPTY_VAL, inplace=True)
 X_Test['State'] = X_Test.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Test.loc[:, 'Date'] = X_Test.Date.dt.strftime("%m%d")
 X_Test["Date"] = X_Test["Date"].astype(int)
 X_Test.head()
train_df = pd.read_csv(sorted(paths)[10])
 test_df = pd.read_csv(sorted(paths)[9])
 submission = pd.read_csv(sorted(paths)[8])
 population = pd.read_csv(sorted(paths)[11])
 country_stats = pd.read_csv(sorted(paths)[7])
 age_by_countries = pd.read_csv(sorted(paths)[1])
 health_index = pd.read_csv(sorted(paths)[3] )<define_variables>
le = preprocessing.LabelEncoder()
X_Train.Country = le.fit_transform(X_Train.Country)
 X_Train['State'] = le.fit_transform(X_Train['State'])
 X_Train.head()
X_Test.Country = le.fit_transform(X_Test.Country)
 X_Test['State'] = le.fit_transform(X_Test['State'])
 X_Test.head()
df_train.head()
df_train.loc[df_train.Country == 'Afghanistan', :]
df_test.tail()
not_in_countries = []
for country in countries:
if country not in population_countries:
not_in_countries.append(country)
 not_in_countries_stats = []
for country in countries:
if country not in stats_countries:
not_in_countries_stats.append(country)
 not_in_countries_age = []
for country in countries:
if country not in age_countries:
not_in_countries_age.append(country)
 not_in_countries_health = []
for country in countries:
if country not in health_countries:
not_in_countries_health.append(country )<define_variables>
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
countries = X_Train.Country.unique()
df_out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_Train.loc[X_Train.Country == country, :].State.unique()
for state in states:
X_Train_CS = X_Train.loc[(X_Train.Country == country)&(X_Train.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_Train_CS = X_Train_CS.loc[:, 'ConfirmedCases']
y2_Train_CS = X_Train_CS.loc[:, 'Fatalities']
X_Train_CS = X_Train_CS.loc[:, ['State', 'Country', 'Date']]
X_Train_CS.Country = le.fit_transform(X_Train_CS.Country)
 X_Train_CS['State'] = le.fit_transform(X_Train_CS['State'])
 X_Test_CS = X_Test.loc[(X_Test.Country == country)&(X_Test.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_Test_CS_Id = X_Test_CS.loc[:, 'ForecastId']
X_Test_CS = X_Test_CS.loc[:, ['State', 'Country', 'Date']]
X_Test_CS.Country = le.fit_transform(X_Test_CS.Country)
 X_Test_CS['State'] = le.fit_transform(X_Test_CS['State'])
 model1 = XGBRegressor(n_estimators=1000)
 model1.fit(X_Train_CS, y1_Train_CS)
 y1_pred = model1.predict(X_Test_CS)
 model2 = XGBRegressor(n_estimators=1000)
 model2.fit(X_Train_CS, y2_Train_CS)
 y2_pred = model2.predict(X_Test_CS)
 df = pd.DataFrame({'ForecastId': X_Test_CS_Id, 'ConfirmedCases': y1_pred, 'Fatalities': y2_pred})
 df_out = pd.concat([df_out, df], axis=0)
stats_countries_map = {'Antigua&Barbuda': 'Antigua and Barbuda',
'Bahamas,The': 'Bahamas',
'Bosnia&Herzegovina': 'Bosnia and Herzegovina',
'BurkinaFaso': 'Burkina Faso',
'CapeVerde': 'Cabo Verde',
'CentralAfricanRep.': 'Central African Republic',
'Congo,Dem.Rep.': 'Congo(Kinshasa)',
'Congo,Repub.ofthe': 'Congo(Brazzaville)',
'CostaRica': 'Costa Rica',
"Coted'Ivoire": "Cote d'Ivoire",
'CzechRepublic': 'Czechia',
'DominicanRepublic': 'Dominican Republic',
'ElSalvador': 'El Salvador',
'EquatorialGuinea': 'Equatorial Guinea',
'Swaziland': 'Eswatini',
'Gambia,The': 'Gambia',
'Korea,South': 'Korea, South',
'NewZealand': 'New Zealand',
'Macedonia': 'North Macedonia',
'PapuaNewGuinea': 'Papua New Guinea',
'SaintKitts&Nevis': 'Saint Kitts and Nevis',
'SaintLucia': 'Saint Lucia',
'SaintVincentandtheGrenadines': 'Saint Vincent and the Grenadines',
'SanMarino': 'San Marino',
'SaudiArabia': 'Saudi Arabia',
'SouthAfrica': 'South Africa',
'SriLanka': 'Sri Lanka',
'Taiwan': 'Taiwan*',
'EastTimor': 'Timor-Leste',
'Trinidad&Tobago': 'Trinidad and Tobago',
'UnitedStates': 'US',
'UnitedArabEmirates': 'United Arab Emirates',
'UnitedKingdom': 'United Kingdom'}
map_state_rev_stat = {k: v for k, v in stats_countries_map.items() }<define_variables>
df_out.ForecastId = df_out.ForecastId.astype('int')
 df_out.tail()
df_out.to_csv('submission.csv', index=False )