Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<create_dataframe>
DATASET_DIR = ".. /input/covid19-global-forecasting-week-2"
TRAIN_FILE = DATASET_DIR + "/train.csv"
TEST_FILE = DATASET_DIR + "/test.csv"
indiv_pivot = pd.pivot_table(indiv,index=["idhogar"],
values=[
'age',
'rez_esc',
'escolari',
'dis',
'female',
'instlevel_argmax',
'indiv_digital',
'tamviv',
'hhsize'
],
aggfunc={
'age':np.sum,
'rez_esc':np.sum,
'escolari':np.sum,
'dis' : np.sum,
'female' : np.sum,
'instlevel_argmax':[np.max,np.min,np.mean] ,
'indiv_digital':[np.max,np.min,np.mean] ,
'tamviv':[np.max,np.min] ,
'hhsize':[np.max,np.min]
},
fill_value=0 )<create_dataframe>
%matplotlib inline
indiv_toframe = pd.DataFrame(indiv_pivot.to_records() )<rename_columns>
df_train = pd.read_csv(TRAIN_FILE)
 df_train.head()
indiv_toframe.columns = [hdr.replace("('", "" ).replace("', '", "_" ).replace("')", "")\
for hdr in indiv_toframe.columns]<feature_engineering>
df_test = pd.read_csv(TEST_FILE)
 df_test.head(5 )
indiv_toframe['mean_age_hhsize'] = indiv_toframe['age_sum'] / indiv_toframe['hhsize_amax']
indiv_toframe['mean_age_tamviv'] = indiv_toframe['age_sum'] / indiv_toframe['tamviv_amax']<feature_engineering>
df_train["Province_State"].isnull().sum()
indiv_toframe['mean_escolari_hhsize'] = indiv_toframe['escolari_sum'] / indiv_toframe['hhsize_amax']
indiv_toframe['mean_escolari_tamviv'] = indiv_toframe['escolari_sum'] / indiv_toframe['tamviv_amax']<feature_engineering>
df_train.isnull().sum()
indiv_toframe['mean_rez_esc_hhsize'] = indiv_toframe['rez_esc_sum'] / indiv_toframe['hhsize_amax']
indiv_toframe['mean_rez_esc_tamviv'] = indiv_toframe['rez_esc_sum'] / indiv_toframe['tamviv_amax']<feature_engineering>
df_train[df_train["Province_State"].notnull() ]
indiv_toframe['mean_female_hhsize'] = indiv_toframe['female_sum'] / indiv_toframe['hhsize_amax']
indiv_toframe['mean_female_tamviv'] = indiv_toframe['female_sum'] / indiv_toframe['tamviv_amax']<feature_engineering>
df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True )
indiv_toframe['mean_dis_hhsize'] = indiv_toframe['dis_sum'] / indiv_toframe['hhsize_amax']
indiv_toframe['mean_dis_tamviv'] = indiv_toframe['dis_sum'] / indiv_toframe['tamviv_amax']<merge>
pd.plotting.register_matplotlib_converters()
grouped_data = df_train.groupby('Date')['Date', 'ConfirmedCases', 'Fatalities'].sum().reset_index()
grouped_data = grouped_data.sort_values(by=['Date'], ascending=True)
 grouped_data['ConfirmedCases'] = grouped_data['ConfirmedCases'].astype(int)
 grouped_data['Fatalities'] = grouped_data['Fatalities'].astype(int)
 grouped_data.head()
head_agg = head.merge(indiv_toframe, on = 'idhogar',
how = 'left' )<prepare_x_and_y>
df_train['Date'] = pd.to_datetime(df_train['Date'])
 grouped_data = df_train.groupby(['Date'],as_index=True ).agg({'ConfirmedCases': 'max','Fatalities': 'max'})
 grouped_data['ConfirmedCases'] = grouped_data['ConfirmedCases'].astype(int)
 grouped_data['Fatalities'] = grouped_data['Fatalities'].astype(int)
 display(grouped_data.head() )
train_labels = np.array(list(head_agg[head_agg['Target'].notnull() ]['Target'].astype(np.uint8)))
 train_set = head_agg[head_agg['Target'].notnull() ].drop(labels = ['Id', 'idhogar', 'Target'], axis = 1)
 test_set = head_agg[head_agg['Target'].isnull() ].drop(labels = ['Id', 'idhogar', 'Target'], axis = 1)
 submission_base = df_test[['Id', 'idhogar']].copy()<choose_model_class>
grouped_data = df_train.groupby('Country_Region')['ConfirmedCases', 'Fatalities'].sum().reset_index()
grouped_data = grouped_data.sort_values(by=['ConfirmedCases'], ascending=False)
 grouped_data['ConfirmedCases'] = grouped_data['ConfirmedCases'].astype(int)
 grouped_data['Fatalities'] = grouped_data['Fatalities'].astype(int)
 grouped_data.head(10 )
sm = SMOTE(random_state = 33 )<train_model>
grouped_data = df_train.groupby('Country_Region')['ConfirmedCases', 'Fatalities'].sum().reset_index()
grouped_data_sort_confirmed_cases = grouped_data.sort_values(by=['ConfirmedCases', 'Fatalities'], ascending=False)[:10]
grouped_data_sort_fatalities = grouped_data.sort_values(by=['Fatalities', 'ConfirmedCases'], ascending=False)[:10]
train_set_balanced, train_labels_balanced = sm.fit_sample(train_set, train_labels )<categorify>
train_data = df_train.copy()
test_data = df_test.copy()
train_data['Date'] = train_data['Date'].dt.strftime("%m%d" ).astype(int)
 test_data['Date'] = test_data['Date'].dt.strftime("%m%d" ).astype(int)
features = list(train_set.columns)
 pipeline = Pipeline([('imputer', Imputer(strategy = 'median')) , 
('scaler', MinMaxScaler())])
 train_set = pipeline.fit_transform(train_set)
 test_set = pipeline.transform(test_set )<categorify>
train_data['Province_State'] = train_data['Province_State'].fillna("N/D")
 test_data['Province_State'] = test_data['Province_State'].fillna("N/D" )
train_set_balanced = pipeline.fit_transform(train_set_balanced )<compute_train_metric>
LE=LabelEncoder()
train_data['Province_State']=LE.fit_transform(train_data['Province_State'])
 test_data['Province_State']=LE.transform(test_data['Province_State'])
 train_data['Country_Region']=LE.fit_transform(train_data['Country_Region'])
 test_data['Country_Region']=LE.transform(test_data['Country_Region'] )
model_no_smote = RandomForestClassifier(n_estimators=100, random_state=10,
n_jobs = -1)
 cv_score_no_smote = cross_val_score(model_no_smote, train_set, train_labels, cv = 10, scoring = scorer)
 print(f'10 Fold Cross Validation F1 Score = {round(cv_score_no_smote.mean() , 4)} with std = {round(cv_score_no_smote.std() , 4)}' )<compute_train_metric>
x_cols = ['Date','Province_State', 'Country_Region']
y_cols = ['ConfirmedCases', 'Fatalities']
train_data[x_cols].head(10 )
model_smote = RandomForestClassifier(n_estimators=100, random_state=10,
n_jobs = -1)
 cv_score_smote = cross_val_score(model_smote, train_set_balanced, train_labels_balanced, cv = 10, scoring = scorer)
 print(f'10 Fold Cross Validation F1 Score = {round(cv_score_smote.mean() , 4)} with std = {round(cv_score_smote.std() , 4)}' )<train_model>
model_dtr = DecisionTreeRegressor(max_depth=None)
 param_grid = {
'n_estimators': [100, 200, 300],
'learning_rate': [0.1, 0.01, 0.001]
}
model_abr = AdaBoostRegressor(base_estimator=model_dtr)
 model = RandomizedSearchCV(estimator = model_abr, param_distributions = param_grid,
n_iter = 100, cv =10, verbose=0, random_state=2020, n_jobs = -1)
 model.fit(train_data[x_cols], train_data[y_cols[0]])
 predictions1 = model.predict(test_data[x_cols])
 model = RandomizedSearchCV(estimator = model_abr, param_distributions = param_grid,
n_iter = 100, cv = 10, verbose=0, random_state=2020, n_jobs = -1)
 model.fit(train_data[x_cols], train_data[y_cols[1]])
 predictions2 = model.predict(test_data[x_cols] )
model_no_smote.fit(train_set, train_labels)
 feature_importances_no_smote = pd.DataFrame({'feature': features, 'importance': model_no_smote.feature_importances_})
 feature_importances_no_smote.head()<train_model>
submission = pd.DataFrame({'ForecastId':test_data['ForecastId'],'ConfirmedCases':predictions1,'Fatalities':predictions2})
 submission['ConfirmedCases'] = submission['ConfirmedCases'].astype(int)
 submission['Fatalities'] = submission['Fatalities'].astype(int)
 submission.head(50 )
model_smote.fit(train_set_balanced, train_labels_balanced)
 feature_importances_smote = pd.DataFrame({'feature': features, 'importance': model_smote.feature_importances_})
 feature_importances_smote.head()<filter>
filename = 'submission.csv'
submission.to_csv(filename,index=False)
 print('Saved file: ' + filename )