Kaggle competition: COVID19 Global Forecasting (Week 1)
model_infected = model_infected.fit(X, y["confirmed"])
 model_deaths = model_deaths.fit(X, y["deaths"] )<load_from_csv>
cleaned_data = pd.read_csv('.. /input/corona-virus-report/covid_19_clean_complete.csv', parse_dates=['Date'])
 cleaned_data.rename(columns={'ObservationDate': 'date',
'Province/State':'state',
'Country/Region':'country',
'Last Update':'last_updated',
'Confirmed': 'confirmed',
'Deaths':'deaths',
'Recovered':'recovered'
}, inplace=True)
 cases = ['confirmed', 'deaths', 'recovered', 'active']
cleaned_data['active'] = cleaned_data['confirmed'] - cleaned_data['deaths'] - cleaned_data['recovered']
cleaned_data['country'] = cleaned_data['country'].replace('Mainland China', 'China')
 cleaned_data[['state']] = cleaned_data[['state']].fillna('')
 cleaned_data[cases] = cleaned_data[cases].fillna(0)
 cleaned_data.rename(columns={'Date':'date'}, inplace=True)
 data = cleaned_data
display(data.head())
 display(data.info() )
test_df = pd.read_csv(".. /input/covid19-global-forecasting-week-1/test.csv")
 test_df.rename(columns={'Date': 'date',
'Province/State':'state',
'Country/Region':'country',
}, inplace=True)
 test_df["date"] = pd.to_datetime(test_df['date'])
 test_df['state'] = test_df['state'].fillna('')
 test_df.info()<merge>
def p2f(x):

 try:
return float(x.strip('%')) /100
except:
return np.nan
def age2int(x):

 try:
return int(x)
 except:
return np.nan
def fert2float(x):

 try:
return float(x)
 except:
return np.nan
countries_df = pd.read_csv(".. /input/population-by-country-2020/population_by_country_2020.csv", converters={'Urban Pop %':p2f,
'Fert.Rate':fert2float,
'Med.Age':age2int})
 countries_df.rename(columns={'Country(or dependency)': 'country',
'Population(2020)' : 'population',
'Density(P/Km²)' : 'density',
'Fert.Rate' : 'fertility',
'Med.Age' : "age",
'Urban Pop %' : 'urban_percentage'}, inplace=True)
 countries_df['country'] = countries_df['country'].replace('United States', 'US')
 countries_df = countries_df[["country", "population", "density", "fertility", "age", "urban_percentage"]]
countries_df.head()
test_df = test_df.merge(df_temperature, on=['country','date', 'state'], how='left')
 test_df = test_df.merge(countries_df, on=['country'], how='left')
 test_df = test_df.merge(icu_cleaned, on=['country'], how='left')
 test_df.shape<predict_on_test>
data = pd.merge(data, countries_df, on='country' )
X_test = test_df.set_index("ForecastId" ).drop(["Lat", "Long", "date", "state", "country", "index"], axis=1 ).fillna(0)
 y_pred_confirmed = model_infected.predict(X_test)
 y_pred_deaths = model_deaths.predict(X_test )<save_to_csv>
formated_gdf = data.groupby(['date', 'country'])['confirmed', 'population'].max()
formated_gdf = formated_gdf.reset_index()
formated_gdf['date'] = pd.to_datetime(formated_gdf['date'])
 formated_gdf['date'] = formated_gdf['date'].dt.strftime('%m/%d/%Y')
 formated_gdf['infectionRate'] = round(( formated_gdf['confirmed']/formated_gdf['population'])*100, 8)
 fig = px.scatter_geo(formated_gdf, locations="country", locationmode='country names',
color="infectionRate", size='infectionRate', hover_name="country",
range_color= [0, 0.2],
projection="natural earth", animation_frame="date",
title='COVID-19: Spread Over Time(Normalized by Country Population)', color_continuous_scale="portland")
 fig.show()
submission = pd.DataFrame()
submission["ForecastId"] = test_df["ForecastId"]
submission = submission.set_index(['ForecastId'])
 submission["ConfirmedCases"] = y_pred_confirmed.astype(int)
 submission["Fatalities"] = y_pred_deaths.astype(int)
 submission.to_csv("submission.csv")
 submission.head()<train_model>
formated_gdf = data.groupby(['date', 'country'])['confirmed', 'deaths'].max()
formated_gdf = formated_gdf.reset_index()
formated_gdf['date'] = pd.to_datetime(formated_gdf['date'])
 formated_gdf['date'] = formated_gdf['date'].dt.strftime('%m/%d/%Y')
 formated_gdf['mortalityRate'] = round(( formated_gdf['deaths']/formated_gdf['confirmed'])*100, 2)
 fig = px.scatter_geo(formated_gdf.fillna(0), locations="country", locationmode='country names',
color="mortalityRate", size='mortalityRate', hover_name="country",
range_color= [0, 10],
projection="natural earth", animation_frame="date",
title='COVID-19: Mortality Rate in % by country', color_continuous_scale="portland")
 fig.show()
m = Prophet()
italy_data = data[data['country']=='Italy']
ts_df = pd.concat([italy_data['date'], np.log(italy_data['confirmed']+1)], axis=1, keys=['ds', 'y'])
 ts_df.head()
m.fit(ts_df )<predict_on_test>
icu_df = pd.read_csv(".. /input/hospital-beds-by-country/API_SH.MED.BEDS.ZS_DS2_en_csv_v2_887506.csv")
 icu_df['Country Name'] = icu_df['Country Name'].replace('United States', 'US')
 icu_df['Country Name'] = icu_df['Country Name'].replace('Russian Federation', 'Russia')
 icu_df['Country Name'] = icu_df['Country Name'].replace('Iran, Islamic Rep.', 'Iran')
 icu_df['Country Name'] = icu_df['Country Name'].replace('Egypt, Arab Rep.', 'Egypt')
 icu_df['Country Name'] = icu_df['Country Name'].replace('Venezuela, RB', 'Venezuela')
 data['country'] = data['country'].replace('Czechia', 'Czech Republic')
future = m.make_future_dataframe(periods=14)
 forecast = m.predict(future)
 forecast[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail()<feature_engineering>
icu_cleaned = pd.DataFrame()
icu_cleaned["country"] = icu_df["Country Name"]
icu_cleaned["icu"] = np.nan
for year in range(1960, 2020):
year_df = icu_df[str(year)].dropna()
icu_cleaned["icu"].loc[year_df.index] = year_df.values
ts_df['infectionRate'] = round(( ts_df['confirmed']/ts_df['population'])*100, 5)
 ts_df = ts_df[ts_df['infectionRate'] >= threshold]
ts_df.index = ts_df.date<drop_column>
data = pd.merge(data, icu_cleaned, on='country' )
ts_df = ts_df.drop([
"country",
"active",
"recovered",
"infectionRate",
"state",
"date",
"Lat",
"Long",
"population",
"density",
"fertility",
"age",
"urban_percentage",
"icu",
"index"
], axis= 1 ).dropna() 
<define_variables>
df_temperature = pd.read_csv(".. /input/covid19-global-weather-data/temperature_dataframe.csv")
 df_temperature['country'] = df_temperature['country'].replace('USA', 'US')
 df_temperature['country'] = df_temperature['country'].replace('UK', 'United Kingdom')
 df_temperature = df_temperature[["country", "province", "date", "humidity", "sunHour", "tempC", "windspeedKmph"]].reset_index()
df_temperature.rename(columns={'province': 'state'}, inplace=True)
 df_temperature["date"] = pd.to_datetime(df_temperature['date'])
 df_temperature['state'] = df_temperature['state'].fillna('')
 df_temperature.info()
train_percentage = 0.75<split>
data = data.merge(df_temperature, on=['country','date', 'state'], how='inner')
 data.to_csv("countries_icu_temp.csv" )
train = ts_df[:int(train_percentage*(len(ts_df)))]
valid = ts_df[int(( 1-train_percentage)*(len(ts_df))):]<train_model>
threshold = 0
train_data['infectionRate'] = round(( train_data['confirmed']/train_data['population'])*100, 5)
 train_data = train_data[train_data['infectionRate'] >= threshold]
print(train_data.shape )
model = VAR(endog=train)
 model_fit = model.fit()<predict_on_test>
train_data = train_data.drop([
"country",
"active",
"recovered",
"infectionRate",
"state",
"Lat",
"Long",
"date",
"index"
], axis= 1 ).dropna()
y = train_data[["confirmed", "deaths"]]
X = train_data.drop(["confirmed", "deaths"],axis=1)
 display(X.head())
 print(X.shape, y.shape )
prediction = model_fit.forecast(model_fit.y, steps=len(valid))<create_dataframe>
from sklearn.preprocessing import StandardScaler
pred = pd.DataFrame(index=range(0,len(prediction)) ,columns=ts_df.columns)
 for j in range(0,prediction.shape[1]):
for i in range(0, len(prediction)) :
pred.iloc[i][j] = prediction[i][j]<compute_test_metric>
def rmsle(y_true, y_pred):

 return np.sqrt(mean_squared_log_error(y_true, y_pred))
rmsle_scorer = make_scorer(rmsle)
 X_train, X_val, y_train, y_val = tts(X, y, test_size= 0.2, random_state=42, shuffle=True )
for i in ts_df.columns:
print('rmse value for', i, 'is : ', np.sqrt(mean_squared_error(pred[i], valid[i])) )<train_model>
model_infected = DecisionTreeRegressor(random_state=42, criterion="mae")
 scores = cross_val_score(model_infected,
X_train,
y_train["confirmed"],
cv=5, scoring=rmsle_scorer)
 print("Cross Validation of Confirmed Cases: Mean = {}, std = {}".format(scores.mean() , scores.std()))
model_infected.fit(X_train, y_train["confirmed"])
 result_infected = rmsle(y_val["confirmed"], model_infected.predict(X_val))
print("Validation Infected set RMSLE: {}".format(result_infected))
days_to_predict = 14
future_dt = pd.date_range(ts_df.last_valid_index() , periods=days_to_predict)
 model = VAR(endog=ts_df)
 model_fit = model.fit()
yhat = model_fit.forecast(model_fit.y, steps=days_to_predict)
 pred_df = pd.DataFrame(yhat, columns=ts_df.columns)
 pred_df = pred_df.drop([
"humidity",
"sunHour",
"tempC",
"windspeedKmph"
], axis=1)
 pred_df['confirmed'] = pred_df['confirmed'].astype(int)
 pred_df['deaths'] = pred_df['deaths'].astype(int)
 pred_df.index = future_dt
pred_df.head()<import_modules>
model_deaths = DecisionTreeRegressor(random_state=42, criterion="mae")
 scores = cross_val_score(model_deaths,
X_train,
y_train["deaths"],
cv=5, scoring=rmsle_scorer)
 print("Cross Validation of Fatal Cases: Mean = {}, std = {}".format(scores.mean() , scores.std()))
model_deaths.fit(X_train, y_train["deaths"])
 result_deaths = rmsle(y_val["deaths"], model_deaths.predict(X_val))
print("Validation Death set RMSLE: {}".format(result_deaths))
import numpy as np
import pandas as pd
import xgboost as xgb
from xgboost import plot_importance, plot_tree
from sklearn.metrics import mean_squared_error, mean_absolute_error
from google.cloud import bigquery<load_from_csv>
print("Final Validatio score: {}".format(np.mean([result_infected, result_deaths])) )
train = pd.read_csv(".. /input/covid19-global-forecasting-week-1/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-1/test.csv" )<create_dataframe>
model_infected = model_infected.fit(X, y["confirmed"])
 model_deaths = model_deaths.fit(X, y["deaths"] )
%%time
client = bigquery.Client()
dataset_ref = client.dataset("noaa_gsod", project="bigquery-public-data")
 dataset = client.get_dataset(dataset_ref)
 tables = list(client.list_tables(dataset))
table_ref = dataset_ref.table("stations")
 table = client.get_table(table_ref)
 stations_df = client.list_rows(table ).to_dataframe()
table_ref = dataset_ref.table("gsod2020")
 table = client.get_table(table_ref)
 twenty_twenty_df = client.list_rows(table ).to_dataframe()
stations_df['STN'] = stations_df['usaf'] + '-' + stations_df['wban']
twenty_twenty_df['STN'] = twenty_twenty_df['stn'] + '-' + twenty_twenty_df['wban']
cols_1 = ['STN', 'mo', 'da', 'temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']
cols_2 = ['STN', 'country', 'state', 'call', 'lat', 'lon', 'elev']
weather_df = twenty_twenty_df[cols_1].join(stations_df[cols_2].set_index('STN'), on='STN')
 weather_df.tail(10 )<feature_engineering>
test_df = pd.read_csv(".. /input/covid19-global-forecasting-week-1/test.csv")
 test_df.rename(columns={'Date': 'date',
'Province/State':'state',
'Country/Region':'country',
}, inplace=True)
 test_df["date"] = pd.to_datetime(test_df['date'])
 test_df['state'] = test_df['state'].fillna('')
 test_df.info()
weather_df['day_from_jan_first'] =(weather_df['da'].apply(int)
 + 31*(weather_df['mo']=='02')
 + 60*(weather_df['mo']=='03')
 + 91*(weather_df['mo']=='04')
)
 mo = train['Date'].apply(lambda x: x[5:7])
 da = train['Date'].apply(lambda x: x[8:10])
 train['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 C = []
for j in train.index:
df = train.iloc[j:(j+1)]
mat = cdist(df[['Lat','Long', 'day_from_jan_first']],
weather_df[['lat','lon', 'day_from_jan_first']],
metric='euclidean')
 new_df = pd.DataFrame(mat, index=df.Id, columns=weather_df.index)
 arr = new_df.values
new_close = np.where(arr == np.nanmin(arr, axis=1)[:,None],new_df.columns,False)
 L = [i[i.astype(bool)].tolist() [0] for i in new_close]
C.append(L[0])
 train['closest_station'] = C
train = train.set_index('closest_station' ).join(weather_df[['temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']], ).reset_index().drop(['index'], axis=1)
 train.sort_values(by=['Id'], inplace=True)
 train.head()<feature_engineering>
test_df = test_df.merge(df_temperature, on=['country','date', 'state'], how='left')
 test_df = test_df.merge(countries_df, on=['country'], how='left')
 test_df = test_df.merge(icu_cleaned, on=['country'], how='left')
 test_df.shape
weather_df['day_from_jan_first'] =(weather_df['da'].apply(int)
 + 31*(weather_df['mo']=='02')
 + 60*(weather_df['mo']=='03')
 + 91*(weather_df['mo']=='04')
)
 mo = test['Date'].apply(lambda x: x[5:7])
 da = test['Date'].apply(lambda x: x[8:10])
 test['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 C = []
for j in test.index:
df = test.iloc[j:(j+1)]
mat = cdist(df[['Lat','Long', 'day_from_jan_first']],
weather_df[['lat','lon', 'day_from_jan_first']],
metric='euclidean')
 new_df = pd.DataFrame(mat, index=df.ForecastId, columns=weather_df.index)
 arr = new_df.values
new_close = np.where(arr == np.nanmin(arr, axis=1)[:,None],new_df.columns,False)
 L = [i[i.astype(bool)].tolist() [0] for i in new_close]
C.append(L[0])
 test['closest_station'] = C
test = test.set_index('closest_station' ).join(weather_df[['temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog']], ).reset_index().drop(['index'], axis=1)
 test.sort_values(by=['ForecastId'], inplace=True)
 test.head()<data_type_conversions>
X_test = test_df.set_index("ForecastId" ).drop(["Lat", "Long", "date", "state", "country", "index"], axis=1 ).fillna(0)
 y_pred_confirmed = model_infected.predict(X_test)
 y_pred_deaths = model_deaths.predict(X_test )
train["wdsp"] = pd.to_numeric(train["wdsp"])
 test["wdsp"] = pd.to_numeric(test["wdsp"] )<data_type_conversions>
submission = pd.DataFrame()
submission["ForecastId"] = test_df["ForecastId"]
submission = submission.set_index(['ForecastId'])
 submission["ConfirmedCases"] = y_pred_confirmed.astype(int)
 submission["Fatalities"] = y_pred_deaths.astype(int)
 submission.to_csv("submission.csv")
 submission.head()
train["fog"] = pd.to_numeric(train["fog"])
 test["fog"] = pd.to_numeric(test["fog"] )<drop_column>
m = Prophet()
italy_data = data[data['country']=='Italy']
ts_df = pd.concat([italy_data['date'], np.log(italy_data['confirmed']+1)], axis=1, keys=['ds', 'y'])
 ts_df.head()
m.fit(ts_df )
X_train = train.drop(["Fatalities", "ConfirmedCases"], axis=1 )<define_variables>
future = m.make_future_dataframe(periods=14)
 forecast = m.predict(future)
 forecast[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail()
countries = X_train["Country/Region"]<drop_column>
ts_df['infectionRate'] = round(( ts_df['confirmed']/ts_df['population'])*100, 5)
 ts_df = ts_df[ts_df['infectionRate'] >= threshold]
ts_df.index = ts_df.date
X_train = X_train.drop(["Id"], axis=1)
 X_test = test.drop(["ForecastId"], axis=1 )<data_type_conversions>
ts_df = ts_df.drop([
"country",
"active",
"recovered",
"infectionRate",
"state",
"date",
"Lat",
"Long",
"population",
"density",
"fertility",
"age",
"urban_percentage",
"icu",
"index"
], axis= 1 ).dropna()
X_train['Date']= pd.to_datetime(X_train['Date'])
 X_test['Date']= pd.to_datetime(X_test['Date'] )<rename_columns>
train_percentage = 0.75
X_train = X_train.set_index(['Date'])
 X_test = X_test.set_index(['Date'] )<feature_engineering>
train = ts_df[:int(train_percentage*(len(ts_df)))]
valid = ts_df[int(( 1-train_percentage)*(len(ts_df))):]
def create_time_features(df):

 df['date'] = df.index
df['hour'] = df['date'].dt.hour
df['dayofweek'] = df['date'].dt.dayofweek
df['quarter'] = df['date'].dt.quarter
df['month'] = df['date'].dt.month
df['year'] = df['date'].dt.year
df['dayofyear'] = df['date'].dt.dayofyear
df['dayofmonth'] = df['date'].dt.day
df['weekofyear'] = df['date'].dt.weekofyear
X = df[['hour','dayofweek','quarter','month','year',
'dayofyear','dayofmonth','weekofyear']]
return X<feature_engineering>
model = VAR(endog=train)
 model_fit = model.fit()
create_time_features(X_train)
 create_time_features(X_test )<drop_column>
prediction = model_fit.forecast(model_fit.y, steps=len(valid))
X_train.drop("date", axis=1, inplace=True)
 X_test.drop("date", axis=1, inplace=True )<load_from_csv>
pred = pd.DataFrame(index=range(0,len(prediction)) ,columns=ts_df.columns)
 for j in range(0,prediction.shape[1]):
for i in range(0, len(prediction)) :
pred.iloc[i][j] = prediction[i][j]
world_happiness_index = pd.read_csv(".. /input/world-bank-datasets/World_Happiness_Index.csv" )<groupby>
for i in ts_df.columns:
print('rmse value for', i, 'is : ', np.sqrt(mean_squared_error(pred[i], valid[i])) )
<drop_column><EOS> 
days_to_predict = 14
future_dt = pd.date_range(ts_df.last_valid_index() , periods=days_to_predict)
 model = VAR(endog=ts_df)
 model_fit = model.fit()
yhat = model_fit.forecast(model_fit.y, steps=days_to_predict)
 pred_df = pd.DataFrame(yhat, columns=ts_df.columns)
 pred_df = pred_df.drop([
"humidity",
"sunHour",
"tempC",
"windspeedKmph"
], axis=1)
 pred_df['confirmed'] = pred_df['confirmed'].astype(int)
 pred_df['deaths'] = pred_df['deaths'].astype(int)
 pred_df.index = future_dt
pred_df.head()