Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<drop_column>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from itertools import cycle, islice
import seaborn as sb
import matplotlib.dates as dates
from datetime import datetime
from sklearn import preprocessing
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
df_train.drop(labels=to_drop,inplace=True,axis=1, errors='ignore' )<feature_engineering>
train = pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-4/test.csv' )
df_train['winPlacePerc'] = y_train<count_unique_values>
curr_date = train['Date'].max()
world_cum_confirmed = sum(train[train['Date'] == curr_date].ConfirmedCases)
 world_cum_fatal = sum(train[train['Date'] == curr_date].Fatalities)
 print('Number of Countires: ', len(train['Country_Region'].unique()))
print('End date in train dset: ', curr_date)
 print('Number of confirmed cases: ', world_cum_confirmed)
 print('Number of fatal cases: ', world_cum_fatal )
def train_validation(df, train_size=0.9):
unique_games = df.matchId.unique()
train_index = round(int(unique_games.shape[0]*train_size))
np.random.seed(42)
 np.random.shuffle(unique_games)
 train_Id = unique_games[:train_index]
validation_Id = unique_games[train_index:]
train = df[df.matchId.isin(train_Id)]
validation = df[df.matchId.isin(validation_Id)]
return train, validation
train, validation = train_validation(df_train)
<define_variables>
top_country_c = train[train['Date'] == curr_date].groupby(['Date','Country_Region'] ).sum().sort_values(['ConfirmedCases'], ascending=False)
 top_country_c.head(20 )
train_weights =(1/train['team_size'])
 validation_weights =(1/validation['team_size'])
<prepare_x_and_y>
top_country_f = train[train['Date'] == curr_date].groupby(['Date','Country_Region'] ).sum().sort_values(['Fatalities'], ascending=False)
 top_country_f.head(20 )
train.drop(labels=to_drop2,inplace=True,axis=1, errors='ignore')
 validation.drop(labels=to_drop2,inplace=True,axis=1, errors='ignore')
 X_train = train.drop(labels=['winPlacePerc'],axis=1)
 X_val = validation.drop(labels=['winPlacePerc'], axis=1)
 y_train = train['winPlacePerc']
y_val = validation['winPlacePerc']
print(str(X_train.shape)+ str(X_val.shape))<set_options>
train['MortalityRate'] = train['Fatalities'] / train['ConfirmedCases']
train['MortalityRate'] = train['MortalityRate'].fillna(0.0 )
del df_train
gc.collect()<train_model>
top_country_m = train[train['Date'] == curr_date].groupby(['Country_Region'] ).sum().sort_values(['MortalityRate'], ascending=False)
 top_country_m.head(10 )
lgbm = LGBMRegressor(objective='mae', n_estimators=4500,
learning_rate=0.03, num_leaves=300,
max_depth=14,
n_jobs=-1, random_state=42, verbose=50)
 lgb_reg = lgbm.fit(X_train, y_train, eval_set=[(X_val, y_val)], eval_metric='mae', early_stopping_rounds=100, verbose=50 )<create_dataframe>
for df in [test]:
df['Date'] = pd.to_datetime(df['Date'], infer_datetime_format=True)
 df['Date'] = pd.to_datetime(df['Date'], infer_datetime_format=True )
pd.DataFrame(sorted(zip(lgbm.feature_importances_, X_train.columns)) )<drop_column>
missed = "NA"
def State(state, country):
if state == missed: return country
return state
del X_train, y_train, X_val, y_val
gc.collect()<load_from_csv>
for df in [train, test]:
df['Province_State'].fillna(missed, inplace=True)
 df['Province_State'] = df.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : State(x['Province_State'], x['Country_Region']), axis=1)
 df.loc[:, 'Date'] = df.Date.dt.strftime("%m%d")
 df["Date"] = df["Date"].astype(int )
df_test = pd.read_csv('.. /input/test_V2.csv', sep=',')
 df_test = reduce_mem_usage(df_test )<categorify>
label_encoder = preprocessing.LabelEncoder()
for df in [train, test]:
df['Country_Region'] = label_encoder.fit_transform(df['Country_Region'])
 df['Province_State'] = label_encoder.fit_transform(df['Province_State'] )
df_test = df_test.assign(team_size=df_test.groupby('groupId' ).groupId.transform('count'))
df_test = df_test.assign(max_team_size=df_test.groupby('matchId' ).team_size.transform('max'))
df_test = df_test.assign(match_size=df_test.groupby('matchId' ).Id.transform('nunique'))
df_test = df_test.assign(team_indicator = df_test.team_size.apply(lambda x: 5 if x>= 5 else x))
df_test = pd.get_dummies(df_test, columns=['team_indicator'])
 dummy_cols = ['team_indicator_{}'.format(i)for i in np.arange(1,6)]
df_test[dummy_cols] = df_test.groupby('matchId')[dummy_cols].transform('mean')
 df_test.loc[df_test.team_indicator_1 >= 0.7, 'game_mode'] = 'solo'
df_test.loc[df_test.team_indicator_2 >= 0.6, 'game_mode'] = 'duo'
df_test.loc[(df_test.team_indicator_3 + df_test.team_indicator_4)>= 0.5, 'game_mode'] = 'squad'
df_test.game_mode = np.where(( df_test.team_indicator_5 >= 0.2), 'custom', df_test.game_mode)
 df_test.game_mode = df_test.game_mode.fillna('custom')
 df_test['total_distance'] = df_test.rideDistance + df_test.swimDistance + df_test.walkDistance
df_test['heals_boosts'] = df_test['heals'] + df_test['boosts']
winPoints_clean = df_test.drop(df_test[df_test['winPoints'] < 1].index)
 winPoints_clean = winPoints_clean['winPoints']
winPoints_mean = winPoints_clean.mean()
df_test['winPoints_imp'] = df_test['winPoints'].replace({0 : winPoints_mean})
 rankPoints_clean = df_test.drop(df_test[df_test['rankPoints'] < 1].index)
 rankPoints_clean = rankPoints_clean['rankPoints']
rankPoints_mean = rankPoints_clean.mean()
df_test['rankPoints_imp'] = df_test['rankPoints'].replace({-1 : rankPoints_mean})
 df_test = pd.get_dummies(df_test, columns=['matchType'])
 matchType_enc = df_test.filter(regex='matchType')
 playersJoined_norm(to_pJnorm, df_test)
 matchDuration_wt(to_matchDwt, df_test)
 df_test = engineer_features(df_test)
 df_test, _, test_columns, test_id = engineer_features2(is_train=False)
 df_test.drop(labels=to_drop_test,inplace=True, axis=1, errors='ignore')
 df_test.drop(labels=to_drop2,inplace=True, axis=1, errors='ignore')
<feature_engineering>
def RF() :
model = RandomForestRegressor(n_estimators = 100)
 return model
def XGB() :
model = XGBRegressor(n_estimators=1700, learning_rate=0.6,
n_job=5)
 return model
def LGBM() :
model = LGBMRegressor(iterations=2)
 return model
df_test = reduce_mem_usage(df_test )<save_to_csv>
sub = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in range(len(unique_countries)) :
current_country_train = train.loc[train['Country_Region'] == unique_countries[country]]
current_country_test = test.loc[test['Country_Region'] == unique_countries[country]]
features = ['Country_Region', 'Province_State', 'Date']
X_train = current_country_train[features].to_numpy()
y1_train = current_country_train[['ConfirmedCases']].to_numpy()
y2_train = current_country_train[['Fatalities']].to_numpy()
X_test = current_country_test[features].to_numpy()
y1_train = y1_train.reshape(-1)
 y2_train = y2_train.reshape(-1)
 model1 = XGB()
model1.fit(X_train, y1_train)
 res_cnf_cls = np.round(model1.predict(X_test))
model2 = XGB()
model2.fit(X_train, y2_train)
 res_fac = np.round(model2.predict(X_test))
current_country_test_Id = current_country_test.loc[:, 'ForecastId']
pred = pd.DataFrame({'ForecastId': current_country_test_Id, 'ConfirmedCases': res_cnf_cls, 'Fatalities': res_fac})
 sub = pd.concat([sub, pred], axis=0 )
results = pd.DataFrame()
results['Id'] = test_id
results['winPlacePerc'] = lgbm.predict(df_test, num_iteration=lgbm.best_iteration_)
 results.winPlacePerc = results.winPlacePerc.clip(0, 1)
 results.to_csv("submission.csv", index=False)
 results.head(20 )<set_options>
sub.ForecastId = sub.ForecastId.astype('int')
 sub.to_csv('submission.csv', index=False)
 sub