Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<compute_test_metric>
warnings.filterwarnings('ignore' )
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True<normalization>
dftrain = pd.read_csv('.. /input/covid19-global-forecasting-week-2/train.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'])
 dftest = pd.read_csv('.. /input/covid19-global-forecasting-week-2/test.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'])
 ppp_tabel = pd.read_csv('.. /input/country-ppp/Country_PPP.csv', sep='\s+')
 ppp_tabel.drop('Id', 1,inplace=True)
 ppp_tabel["Country"].replace('_',' ', regex=True,inplace=True)
 ppp_tabel["Country"].replace('United States','US', regex=True,inplace=True)
 ppp_tabel.rename(columns={'Country':'Country_Region'},inplace=True)
 ppp_tabel.sort_values('Country_Region',inplace=True)
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
<train_model>
dftrain['Dayofyear'] = dftrain['Date'].dt.dayofyear
dftest['Dayofyear'] = dftest['Date'].dt.dayofyear
dftest['Expo'] = dftest['Dayofyear']-89.5
dftest = dftest.merge(dftrain[['Country_Region','Province_State','Date','ConfirmedCases','Fatalities']], on=['Country_Region','Province_State','Date'], how='left', indicator=True)
 print("dftest columns =",dftest.columns)
 dftest.head(60)
def model_gbm(features, labels, test_features, test_ids,
nfolds = 5, return_preds = False, hyp = None,random_state_int=101,early_stopping=300):

 feature_names = list(features.columns)
 if hyp is not None:
if 'n_estimators' in hyp:
del hyp['n_estimators']
params = hyp
else:
params = {
'colsample_bytree': 0.88,
'learning_rate': 0.028,
'min_child_samples': 10,
'num_leaves': 36,
'subsample': 0.54,
'class_weight': 'balanced'}
model = lgb.LGBMClassifier(**params, objective = 'multiclass',
n_jobs = 4, n_estimators = 10000,
random_state = 10)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True, random_state= random_state_int)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds =early_stopping,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
callbacks=[lgb.reset_parameter(learning_rate=learning_rate_power_0997)])
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores<init_hyperparams>
grouped=dftrain.groupby(['Country_Region','Province_State'] ).tail(8)
 grouped=grouped.groupby(['Country_Region','Province_State'] ).head(4)
 grouped.drop(['FatalityBasis'],axis=1,inplace=True)
 to_sum = ['NewCases','NewFatalities']
grouped1 = grouped.groupby(['Country_Region'])[to_sum].sum()
grouped1.rename(columns={'NewCases':'NewCases1','NewFatalities':'NewFatalities1'}, inplace=True)
 print("grouped1 columns =",grouped1.columns)
 print("grouped2 columns =",grouped2.columns)
 grouped = pd.merge(grouped1, grouped2, on=['Country_Region'])
 grouped['CasesIncreasePct'] = 100*(grouped['NewCases2']/grouped['NewCases1']-1)
 mask = grouped['CasesIncreasePct'] > 140
grouped.loc[mask,'CasesIncreasePct'] = 140
mask = grouped['CasesIncreasePct'] < 0
grouped.loc[mask,'CasesIncreasePct'] = 0
mask = grouped['CasesIncreasePct'].isnull()
grouped.loc[mask,'CasesIncreasePct'] = 0
grouped['Factor'] =(grouped['CasesIncreasePct']/100+1)**0.25
grouped = pd.merge(grouped, ppp_tabel, on=['Country_Region'])
 grouped['ppp'] = grouped['ppp']/10000.
 mask =(grouped['FatalityPct2'] > 9)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 5
mask =(grouped['FatalityPct2'] < 5)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 5
mask =(grouped['FatalityPct2'] > 6)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 6
mask =(grouped['FatalityPct2'] < 1.5)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 1.5
mask =(grouped['FatalityPct2'] >(9.5 - 0.43*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(9.5 - 0.43*grouped['ppp'])
 mask =(grouped['FatalityPct2'] <(5.6 - 0.5*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(5.6 - 0.5*grouped['ppp'])
 mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 7
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 4
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(7.5 - 0.5*grouped['ppp'])
 grouped.tail(6)
 print("grouped columns =",grouped.columns )
hyp1 = {
'colsample_bytree': 0.88,
'learning_rate': 0.08,
'min_child_samples': 90, 'num_leaves': 34, 'subsample': 0.94, 'reg_lambda': 0.5,
'class_weight': 'balanced'}
hyp2 = {
'colsample_bytree': 0.88,
'learning_rate': 0.08,
'min_child_samples': 90, 'num_leaves': 14, 'subsample': 0.94, 'reg_lambda': 0.5,
'class_weight': 'balanced'}
hyp3 = {
'colsample_bytree': 0.78,
'learning_rate': 0.08,
'min_child_samples': 45, 'num_leaves': 14, 'subsample': 0.64, 'reg_lambda': 0.1,
'class_weight': 'balanced'}
hyp4 = {
'colsample_bytree': 0.72,
'learning_rate': 0.08,
'min_child_samples': 30, 'num_leaves': 18, 'subsample': 0.64, 'reg_lambda': 0.1,
'class_weight': 'balanced'}<compute_train_metric>
dftest.drop('_merge',axis=1,inplace= True)
 dftest = dftest.merge(grouped[['Country_Region','FatalityPct2','Factor']], on=['Country_Region'], how='left')
 dftest = dftest.merge(grouped_gem[['Province_State','Country_Region','ConfirmedCases_base','ConfirmedCases_init','NewCases_base','Fatalities_init','FatalityBasis']], on=['Province_State','Country_Region'], how='left')
%%capture --no-display
submission1, gbm_fi_selected, valid_scores_selected = model_gbm(train_set, train_labels, test_set, test_ids,hyp = hyp1,random_state_int=103,early_stopping=300)
 submission2, gbm_fi_selected, valid_scores_selected = model_gbm(train_set, train_labels, test_set, test_ids,hyp = hyp1,random_state_int=103,early_stopping=300)
 submission3, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels, test_selected, test_ids)
<train_model>
dftest['ConfirmedCases_shift'] = dftest.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases']].transform(lambda x: x.shift(1))
mask = dftest['ConfirmedCases'].isnull()
dftest.loc[mask,'NewCases'] = dftest.loc[mask,'NewCases_base']*(dftest.loc[mask,'Factor']**dftest.loc[mask,'Expo'])
 dftest['NewCases_cum'] = dftest.groupby(['Country_Region', 'Province_State'])[['NewCases']].cumsum()
dftest.loc[mask,'ConfirmedCases'] = dftest.loc[mask,'ConfirmedCases_init'] + dftest.loc[mask,'NewCases_cum']
mask3 = dftest['ConfirmedCases'] > 400000
dftest.loc[mask3,'FatalityPct2'] = dftest.loc[mask3,'FatalityPct2']*0.7
mask4 = dftest['ConfirmedCases'] > 800000
dftest.loc[mask4,'FatalityPct2'] = dftest.loc[mask4,'FatalityPct2']*0.7
dftest['FatalityBasis'] = dftest.groupby(['Country_Region', 'Province_State'])[
['ConfirmedCases']].transform(lambda x: x.shift(10))
dftest.loc[mask,'NewFatalities'] = dftest.loc[mask,'FatalityBasis'] * dftest.loc[mask,'FatalityPct2']/100
mask2 = dftest['NewFatalities'] >1000
dftest.loc[mask2,'NewFatalities'] = 1000
print("MASK2",mask2.sum())
 dftest['NewFatalities_cum'] = dftest.groupby(['Country_Region', 'Province_State'])[['NewFatalities']].cumsum()
dftest.loc[mask,'Fatalities'] = dftest.loc[mask,'Fatalities_init'] + dftest.loc[mask,'NewFatalities_cum']
print("dftest columns =",dftest.columns)
 dftest.head(5 )
def model_gbm_2(features, labels, test_features, test_ids,
nfolds = 5, return_preds = False, hyp = None,random_state_int=101,early_stopping=300):

 feature_names = list(features.columns)
 if hyp is not None:
if 'n_estimators' in hyp:
del hyp['n_estimators']
params = hyp
else:
params = {
'colsample_bytree': 0.88,
'learning_rate': 0.028,
'min_child_samples': 10,
'num_leaves': 36,
'subsample': 0.54,
'class_weight': 'balanced'}
model = lgb.LGBMClassifier(**params, objective = 'multiclass',
n_jobs = 4, n_estimators = 10000,
random_state = 10)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True, random_state= random_state_int)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds =early_stopping,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
verbose = 200)
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores<compute_train_metric>
dftest.drop(['Dayofyear',
'Expo','FatalityPct2', 'Factor',
'ConfirmedCases_base', 'ConfirmedCases_init',
'NewCases_base', 'Fatalities_init', 'FatalityBasis',
'ConfirmedCases_shift',
'NewCases', 'NewCases_cum', 'NewFatalities','NewFatalities_cum'],axis=1,inplace=True)
 final = dftest.groupby(['Country_Region','Province_State'] ).tail(1)
 dftest.drop(['Province_State'],axis=1,inplace=True)
 dftest.rename(columns={'Province_State_orig':'Province_State'},inplace=True )
%%capture --no-display
submission4, gbm_fi_selected, valid_scores_selected = model_gbm_2(train_set, train_labels, test_set, test_ids,hyp = hyp3,random_state_int=103,early_stopping=300 )<merge>
dftest.drop(['Province_State','Country_Region','Date'],axis=1,inplace=True)
 print("dftest columns =",dftest.columns)
<feature_engineering><EOS> 
dftest.ForecastId = dftest.ForecastId.astype('int')
 dftest['ConfirmedCases'] = dftest['ConfirmedCases'].round().astype(int)
 dftest['Fatalities'] = dftest['Fatalities'].round().astype(int)
 dftest.to_csv('submission.csv', index=False)