Kaggle competition: COVID19 Global Forecasting (Week 1)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-1)<merge>
import numpy as np
import pandas as pd
import xgboost as xgb
from xgboost import plot_importance, plot_tree
from sklearn.metrics import mean_squared_error, mean_absolute_error
from google.cloud import bigquery
X_train = pd.merge(left=X_train, right=malaria_world_health, how='left', left_on='Country/Region', right_on='Country')
 X_test = pd.merge(left=X_test, right=malaria_world_health, how='left', left_on='Country/Region', right_on='Country' )<drop_column>
train = pd.read_csv(".. /input/covid19-global-forecasting-week-1/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-1/test.csv" )
X_train.drop("Country", axis=1, inplace=True)
 X_test.drop("Country", axis=1, inplace=True )<load_from_csv>
%%time
client = bigquery.Client()
dataset_ref = client.dataset("noaa_gsod", project="bigquery-public-data")
 dataset = client.get_dataset(dataset_ref)
 tables = list(client.list_tables(dataset))
table_ref = dataset_ref.table("stations")
 table = client.get_table(table_ref)
 stations_df = client.list_rows(table ).to_dataframe()
table_ref = dataset_ref.table("gsod2020")
 table = client.get_table(table_ref)
 twenty_twenty_df = client.list_rows(table ).to_dataframe()
stations_df['STN'] = stations_df['usaf'] + '-' + stations_df['wban']
twenty_twenty_df['STN'] = twenty_twenty_df['stn'] + '-' + twenty_twenty_df['wban']
cols_1 = ['STN', 'mo', 'da', 'temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog', 'slp']
cols_2 = ['STN', 'country', 'state', 'call', 'lat', 'lon', 'elev']
weather_df = twenty_twenty_df[cols_1].join(stations_df[cols_2].set_index('STN'), on='STN')
 weather_df.tail(10 )
human_development_index = pd.read_csv(".. /input/world-bank-datasets/Human_Development_Index.csv" )<merge>
weather_df['day_from_jan_first'] =(weather_df['da'].apply(int)
 + 31*(weather_df['mo']=='02')
 + 60*(weather_df['mo']=='03')
 + 91*(weather_df['mo']=='04')
)
 mo = train['Date'].apply(lambda x: x[5:7])
 da = train['Date'].apply(lambda x: x[8:10])
 train['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 C = []
for j in train.index:
df = train.iloc[j:(j+1)]
mat = cdist(df[['Lat','Long', 'day_from_jan_first']],
weather_df[['lat','lon', 'day_from_jan_first']],
metric='euclidean')
 new_df = pd.DataFrame(mat, index=df.Id, columns=weather_df.index)
 arr = new_df.values
new_close = np.where(arr == np.nanmin(arr, axis=1)[:,None],new_df.columns,False)
 L = [i[i.astype(bool)].tolist() [0] for i in new_close]
C.append(L[0])
 train['closest_station'] = C
train = train.set_index('closest_station' ).join(weather_df[['temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog', 'slp']], ).reset_index().drop(['index'], axis=1)
 train.sort_values(by=['Id'], inplace=True)
 train.head()
X_train = pd.merge(left=X_train, right=human_development_index, how='left', left_on='Country/Region', right_on='Country')
 X_test = pd.merge(left=X_test, right=human_development_index, how='left', left_on='Country/Region', right_on='Country' )<drop_column>
weather_df['day_from_jan_first'] =(weather_df['da'].apply(int)
 + 31*(weather_df['mo']=='02')
 + 60*(weather_df['mo']=='03')
 + 91*(weather_df['mo']=='04')
)
 mo = test['Date'].apply(lambda x: x[5:7])
 da = test['Date'].apply(lambda x: x[8:10])
 test['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 C = []
for j in test.index:
df = test.iloc[j:(j+1)]
mat = cdist(df[['Lat','Long', 'day_from_jan_first']],
weather_df[['lat','lon', 'day_from_jan_first']],
metric='euclidean')
 new_df = pd.DataFrame(mat, index=df.ForecastId, columns=weather_df.index)
 arr = new_df.values
new_close = np.where(arr == np.nanmin(arr, axis=1)[:,None],new_df.columns,False)
 L = [i[i.astype(bool)].tolist() [0] for i in new_close]
C.append(L[0])
 test['closest_station'] = C
test = test.set_index('closest_station' ).join(weather_df[['temp', 'min', 'max', 'stp', 'wdsp', 'prcp', 'fog','slp']], ).reset_index().drop(['index'], axis=1)
 test.sort_values(by=['ForecastId'], inplace=True)
 test.head()
X_train.drop(["Country", "Gross national income(GNI)per capita 2018"], axis=1, inplace=True)
 X_test.drop(["Country", "Gross national income(GNI)per capita 2018"], axis=1, inplace=True )<load_from_csv>
train["wdsp"] = pd.to_numeric(train["wdsp"])
 test["wdsp"] = pd.to_numeric(test["wdsp"] )
night_ranger_predictors = pd.read_csv(".. /input/covid19-demographic-predictors/covid19_by_country.csv" )<drop_column>
train["fog"] = pd.to_numeric(train["fog"])
 test["fog"] = pd.to_numeric(test["fog"] )
night_ranger_predictors = night_ranger_predictors[night_ranger_predictors.Country != "Georgia"]<merge>
X_train = train.drop(["Fatalities", "ConfirmedCases"], axis=1 )
X_train = pd.merge(left=X_train, right=night_ranger_predictors, how='left', left_on='Country/Region', right_on='Country')
 X_test = pd.merge(left=X_test, right=night_ranger_predictors, how='left', left_on='Country/Region', right_on='Country' )<drop_column>
countries = X_train["Country/Region"]
X_train.drop(["Country", "Restrictions", "Quarantine", "Schools","Total Infected", "Total Deaths"], axis=1, inplace=True)
 X_test.drop(["Country", "Restrictions", "Quarantine", "Schools","Total Infected", "Total Deaths"], axis=1, inplace=True )<categorify>
X_train = X_train.drop(["Id"], axis=1)
 X_test = test.drop(["ForecastId"], axis=1 )
X_train = pd.concat([X_train,pd.get_dummies(X_train['Province/State'], prefix='ps')],axis=1)
 X_train.drop(['Province/State'],axis=1, inplace=True)
 X_test = pd.concat([X_test,pd.get_dummies(X_test['Province/State'], prefix='ps')],axis=1)
 X_test.drop(['Province/State'],axis=1, inplace=True )<categorify>
X_train['Date']= pd.to_datetime(X_train['Date'])
 X_test['Date']= pd.to_datetime(X_test['Date'] )
X_train = pd.concat([X_train,pd.get_dummies(X_train['Country/Region'], prefix='cr')],axis=1)
 X_train.drop(['Country/Region'],axis=1, inplace=True)
 X_test = pd.concat([X_test,pd.get_dummies(X_test['Country/Region'], prefix='cr')],axis=1)
 X_test.drop(['Country/Region'],axis=1, inplace=True )<prepare_x_and_y>
X_train = X_train.set_index(['Date'])
 X_test = X_test.set_index(['Date'] )
y_train = train["Fatalities"]<choose_model_class>
reg = xgb.XGBRegressor(n_estimators=1000, subsample=0.5 )<train_model>
world_happiness_index = pd.read_csv(".. /input/world-bank-datasets/World_Happiness_Index.csv" )
reg.fit(X_train, y_train, verbose=True )<prepare_x_and_y>
world_happiness_grouped = world_happiness_index.groupby('Country name' ).nth(-1 )
y_train=train.Fatalities.applymap(lambda x: np.log(x+1))
y_train.Fatalities[y_train.Fatalities.isnull() ] = -1<categorify>
world_happiness_grouped.drop("Year", axis=1, inplace=True )
y_train = y_train.replace(np.nan, -1 )<define_variables>
X_train = pd.merge(left=X_train, right=world_happiness_grouped, how='left', left_on='Country/Region', right_on='Country name')
 X_test = pd.merge(left=X_test, right=world_happiness_grouped, how='left', left_on='Country/Region', right_on='Country name' )
y_train = y_train.replace(np.inf, -1 )<choose_model_class>
malaria_world_health = pd.read_csv(".. /input/world-bank-datasets/Malaria_World_Health_Organization.csv" )
reg = xgb.XGBRegressor(n_estimators=1000, subsample=0.5 )<train_model>
X_train = pd.merge(left=X_train, right=malaria_world_health, how='left', left_on='Country/Region', right_on='Country')
 X_test = pd.merge(left=X_test, right=malaria_world_health, how='left', left_on='Country/Region', right_on='Country' )
reg.fit(X_train, y_train, verbose=True )<prepare_x_and_y>
X_train.drop("Country", axis=1, inplace=True)
 X_test.drop("Country", axis=1, inplace=True )
y_train = train["ConfirmedCases"]<choose_model_class>
human_development_index = pd.read_csv(".. /input/world-bank-datasets/Human_Development_Index.csv" )
reg = xgb.XGBRegressor(n_estimators=1000, subsample=0.5 )<train_model>
X_train = pd.merge(left=X_train, right=human_development_index, how='left', left_on='Country/Region', right_on='Country')
 X_test = pd.merge(left=X_test, right=human_development_index, how='left', left_on='Country/Region', right_on='Country' )
reg.fit(X_train, y_train, verbose=True )<prepare_x_and_y>
X_train.drop(["Country", "Gross national income(GNI)per capita 2018"], axis=1, inplace=True)
 X_test.drop(["Country", "Gross national income(GNI)per capita 2018"], axis=1, inplace=True )
y_train=train.ConfirmedCases.applymap(lambda x: np.log(x+1))
y_train.ConfirmedCases[y_train.ConfirmedCases.isnull() ] = -1<categorify>
night_ranger_predictors = pd.read_csv(".. /input/covid19-demographic-predictors/covid19_by_country.csv" )
y_train = y_train.replace(np.nan, -1 )<define_variables>
night_ranger_predictors = night_ranger_predictors[night_ranger_predictors.Country != "Georgia"]
y_train = y_train.replace(np.inf, -1 )<choose_model_class>
X_train = pd.merge(left=X_train, right=night_ranger_predictors, how='left', left_on='Country/Region', right_on='Country')
 X_test = pd.merge(left=X_test, right=night_ranger_predictors, how='left', left_on='Country/Region', right_on='Country' )
reg = xgb.XGBRegressor(n_estimators=1000, subsample=0.5 )<train_model>
X_train.drop(["Country", "Restrictions", "Quarantine", "Schools","Total Infected", "Total Deaths"], axis=1, inplace=True)
 X_test.drop(["Country", "Restrictions", "Quarantine", "Schools","Total Infected", "Total Deaths"], axis=1, inplace=True )
reg.fit(X_train, y_train, verbose=True )<predict_on_test>
X_train = pd.concat([X_train,pd.get_dummies(X_train['Province/State'], prefix='ps')],axis=1)
 X_train.drop(['Province/State'],axis=1, inplace=True)
 X_test = pd.concat([X_test,pd.get_dummies(X_test['Province/State'], prefix='ps')],axis=1)
 X_test.drop(['Province/State'],axis=1, inplace=True )
y_train = train["ConfirmedCases"]
confirmed_reg = xgb.XGBRegressor(n_estimators=1000, subsample=0.5)
 confirmed_reg.fit(X_train, y_train, verbose=True)
 preds = confirmed_reg.predict(X_test)
 preds = preds.applymap(lambda x: np.exp(x)-1)
 preds = np.array(preds)
 preds[preds < 0] = 0
preds = np.round(preds, 0 )<prepare_output>
X_train = pd.concat([X_train,pd.get_dummies(X_train['Country/Region'], prefix='cr')],axis=1)
 X_train.drop(['Country/Region'],axis=1, inplace=True)
 X_test = pd.concat([X_test,pd.get_dummies(X_test['Country/Region'], prefix='cr')],axis=1)
 X_test.drop(['Country/Region'],axis=1, inplace=True )
preds = np.array(preds )<load_from_csv>
y_train = train["Fatalities"]
submissionOrig = pd.read_csv(".. /input/covid19-global-forecasting-week-1/submission.csv" )<prepare_output>
reg = xgb.XGBRegressor(n_estimators=1000, subsample=0.5 )
submissionOrig["ConfirmedCases"]=pd.Series(preds )<feature_engineering>
reg.fit(X_train, y_train, verbose=True )
for index, row in submissionOrig.iterrows() :
if index >= 1:
if submissionOrig.iloc[index, 'ConfirmedCases'] < submissionOrig.iloc[index - 1, 'ConfirmedCases']:
submissionOrig.at[index, 'ConfirmedCases'] = submissionOrig.iloc[index - 1,'ConfirmedCases']<predict_on_test>
y_train=train.Fatalities.applymap(lambda x: np.log(x+1))
y_train.Fatalities[y_train.Fatalities.isnull() ] = -1
y_train = train["Fatalities"]
confirmed_reg = xgb.XGBRegressor(n_estimators=1000, subsample=0.5)
 confirmed_reg.fit(X_train, y_train, verbose=True)
 preds = confirmed_reg.predict(X_test)
 preds = preds.applymap(lambda x: np.exp(x)-1)
 preds = np.array(preds)
 preds[preds < 0] = 0
preds = np.round(preds, 0)
 submissionOrig["Fatalities"]=pd.Series(preds )<feature_engineering>
y_train = y_train.replace(np.nan, -1 )
for index, row in submissionOrig.iterrows() :
if index >= 1:
if submissionOrig.iloc[index, 'Fatalities'] < submissionOrig.iloc[index - 1, 'Fatalities']:
submissionOrig.at[index, 'Fatalities'] = submissionOrig.iloc[index - 1,'Fatalities']<save_to_csv>
y_train = y_train.replace(np.inf, -1 )
submissionOrig.to_csv('submission.csv',index=False )<import_modules>
reg = xgb.XGBRegressor(n_estimators=1000, subsample=0.5 )
import pandas as pd
from sklearn.tree import DecisionTreeClassifier<load_from_csv>
reg.fit(X_train, y_train, verbose=True )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<predict_on_test>
y_train = train["ConfirmedCases"]
test.head()<drop_column>
reg = xgb.XGBRegressor(n_estimators=1000, subsample=0.5 )
train = train.drop(["Name", "Ticket", "Cabin"], axis=1)
 test = test.drop(["Name", "Ticket", "Cabin"], axis=1 )<categorify>
reg.fit(X_train, y_train, verbose=True )
new_data_train = pd.get_dummies(train)
 new_data_test = pd.get_dummies(test )<data_type_conversions>
y_train=train.ConfirmedCases.applymap(lambda x: np.log(x+1))
y_train.ConfirmedCases[y_train.ConfirmedCases.isnull() ] = -1
new_data_train["Age"].fillna(new_data_train["Age"].mean() , inplace=True)
 new_data_test["Age"].fillna(new_data_test["Age"].mean() , inplace=True )<data_type_conversions>
y_train = y_train.replace(np.nan, -1 )
new_data_test["Fare"].fillna(new_data_test["Fare"].mean() , inplace=True )<prepare_x_and_y>
y_train = y_train.replace(np.inf, -1 )
X = new_data_train.drop("Survived", axis=1)
 y = new_data_train["Survived"]<train_model>
reg = xgb.XGBRegressor(n_estimators=1000, subsample=0.5 )
tree = DecisionTreeClassifier(max_depth = 10, random_state = 0)
 tree.fit(X, y )<import_modules>
reg.fit(X_train, y_train, verbose=True )
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split<drop_column>
y_train = train["ConfirmedCases"]
confirmed_reg = xgb.XGBRegressor(n_estimators=1000, subsample=0.5)
 confirmed_reg.fit(X_train, y_train, verbose=True)
 preds = confirmed_reg.predict(X_test)
 preds = preds.applymap(lambda x: np.exp(x)-1)
 preds = np.array(preds)
 preds[preds < 0] = 0
preds = np.round(preds, 0 )
Xtest = new_data_test
Xtest.head()<split>
preds = np.array(preds )
Xtrain, Xvalidation, Ytrain, Yvalidation = train_test_split(X, y, test_size=0.2, random_state=True )<train_model>
submissionOrig = pd.read_csv(".. /input/covid19-global-forecasting-week-1/submission.csv" )
model = RandomForestClassifier(n_estimators=100,
max_leaf_nodes=12,
max_depth=12,
random_state=0)
 model.fit(Xtrain, Ytrain)
<predict_on_test>
submissionOrig["ConfirmedCases"]=pd.Series(preds )
Yprediction = model.predict(Xvalidation)
 accuracy_score(Yvalidation, Yprediction )<save_to_csv>
for index, row in submissionOrig.iterrows() :
if index >= 1:
if submissionOrig.iloc[index, 'ConfirmedCases'] < submissionOrig.iloc[index - 1, 'ConfirmedCases']:
submissionOrig.at[index, 'ConfirmedCases'] = submissionOrig.iloc[index - 1,'ConfirmedCases']
submission = pd.DataFrame()
submission["PassengerId"] = Xtest["PassengerId"]
submission["Survived"] = model.predict(Xtest)
 submission.to_csv("submission.csv", index=False )<import_modules>
y_train = train["Fatalities"]
confirmed_reg = xgb.XGBRegressor(n_estimators=1000, subsample=0.5)
 confirmed_reg.fit(X_train, y_train, verbose=True)
 preds = confirmed_reg.predict(X_test)
 preds = preds.applymap(lambda x: np.exp(x)-1)
 preds = np.array(preds)
 preds[preds < 0] = 0
preds = np.round(preds, 0)
 submissionOrig["Fatalities"]=pd.Series(preds )
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
for index, row in submissionOrig.iterrows() :
if index >= 1:
if submissionOrig.iloc[index, 'Fatalities'] < submissionOrig.iloc[index - 1, 'Fatalities']:
submissionOrig.at[index, 'Fatalities'] = submissionOrig.iloc[index - 1,'Fatalities']
df_train=pd.read_csv('.. /input/titanic/train.csv')
 df_test=pd.read_csv('.. /input/titanic/test.csv' )<define_variables>
submissionOrig.to_csv('submission.csv',index=False )