Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<save_to_csv>
%matplotlib inline
def sub_pred(preds,df_test,name='submission.csv'):
sub_df = pd.DataFrame({'ID_code':df_test['ID_code'],'target':preds})
 sub_df.to_csv(name, index=False)
 sub_file = sub_name +'.csv'
sub_pred(predictions['target'],df_test,name=sub_file)
 print(sub_file+' --submitted successfully')
 print('Submitting Combination File.. ')
 sub_pred(comb_approach_test,df_test,name='comb_submission.csv')
 print('comb_submission.csv --submitted successfully')
 print('Submitting non-cv File.. ')
 sub_pred(predictions['target_noncv'] ,df_test,name='lgbm_noncv_submission.csv')
 print('lgbm_noncv_submission.csv --submitted successfully')
 print('Submitting non-cv File.. ')
 sub_pred(predictions['target_keras'] ,df_test,name='nn_submission.csv')
 print('nn_submission.csv --submitted successfully' )<statistical_test>
PATH ='/kaggle/input/covid19-global-forecasting-week-4'
def get_bps_from_tree(tree_obj):
idx = np.arange(0, tree_obj.tree_.feature.shape[0])
 idx = list(filter(lambda x: tree_obj.tree_.feature[x] == 0, idx))
bps = sorted(tree_obj.tree_.threshold[idx])
 return bps
def discretize_feature2(df, var_col, label_col,nn,
max_depth=10,
max_leaf_nodes=5,
min_samples_leaf=0.05):
tree = DecisionTreeClassifier(max_depth=max_depth,
max_leaf_nodes=nn,
min_samples_leaf=min_samples_leaf)
 tree = tree.fit(df[[var_col]].as_matrix() , df[label_col].values)
 bps = get_bps_from_tree(tree)
 return bps
def baddist(df,y,var,nn=2,bins=None):
if bins is None:
df['Bin']=df[var]
else:
df['Bin']=pd.cut(df[var],bins ).astype('str')
 if is_string_dtype(df[var]):
df.loc[df['Bin'].isnull() ,'Bin']="null"
if len(df[var].unique())>nn and is_numeric_dtype(df[var]):
if bins is None:
bb=discretize_feature2(df=df, var_col=var, label_col=y,nn=nn)
 bb=[math.floor(f*10000)/10000 for f in bb]
bins=[float('-inf')]+bb+[float('Inf')]
df['Bin']=pd.cut(df[var],bins,duplicates='drop' ).astype('str')
 tab=pd.crosstab(df['Bin'],df[y])
 tab=tab.reset_index()
try:
tab=tab.sort_values(by=['Bin'])
 tab['min']=tab['Bin'].apply(lambda x: x.split(',')[0] ).apply(lambda x: x[1:] ).astype('float')
 tab['max']=tab['Bin'].apply(lambda x: x.split(',')[1] ).apply(lambda x: x[:-1] ).astype('float')
 tab=tab.sort_values('max')
 del tab['min'],tab['max']
except:
tab=tab
del df['Bin']
tab['0%']=round(tab[0]/tab[0].sum() ,4)*100
tab['1%']=round(tab[1]/tab[1].sum() ,4)*100
tab['All']=tab[0]+tab[1]
tab['All%']=round(tab['All']/tab['All'].sum() ,4)*100
tab['Bad Rate%']=round(tab[1]/tab['All'],4)*100
tab['IV']=np.log(tab['1%']/tab['0%'])*(tab['1%']-tab['0%'])
 print(var+' IV after binning：',round(tab['IV'].sum() ,4),"%")
 ivdict={"Var":[var],"IV":[round(tab['IV'].sum() ,6)]}
ivdf=pd.DataFrame.from_dict(ivdict)
 return tab,df,ivdf
def lgbmodelling(features,train_df,test_df,param=None):
print(len(features))
if param is None:
param ={'bagging_fraction': 0.5,
'bagging_freq': 1,
'bagging_seed': 1993,
'feature_fraction': 0.05,
'learning_rate': 0.025,
'metric': 'auc',
'min_data_in_leaf': 50,
'num_leaves': 3,
'objective': 'binary'}
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1993)
 target = train_df['target']
print(len(target))
oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values, target.values)) :
print("Fold {}".format(fold_+1))
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx])
 clf = lgb.train(param, trn_data, num_boost_round=500000, valid_sets = [trn_data, val_data], verbose_eval=5000,
early_stopping_rounds = 3000)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.6f}".format(roc_auc_score(target, oof)))
 return oof,predictions 
<load_from_csv>
train = pd.read_csv(f'{PATH}/train.csv')
 train.head()
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<create_dataframe>
total_countries_covid = train.groupby(['Country_Region'])['ConfirmedCases'].sum()
features = ['var_'+str(c)for c in range(0,200)]
ivdf=pd.DataFrame()
for f in features:
_,_,iv=baddist(df=train_df, y='target',var=f,nn=10)
 ivdf=pd.concat([ivdf,iv])
 ivdf=ivdf.sort_values('IV',ascending=False)
 ivdf<feature_engineering>
df = train.fillna('NA' ).groupby(['Country_Region','Province_State','Date'])['ConfirmedCases'].sum() \ 
.groupby(['Country_Region','Province_State'] ).max().sort_values() \ 
.groupby(['Country_Region'] ).sum().sort_values(ascending = False)
 top20_countries = pd.DataFrame(df ).head(20)
 top20_countries
features = list(ivdf.loc[ivdf['IV']>3,:]['Var'])
 print(features)
 train_df['pred_lgb_ivgt3'],test_df['pred_lgb_ivgt3']=lgbmodelling(features,train_df,test_df)
 features = list(ivdf.loc[ivdf['IV']<=3,:]['Var'])
 print(features)
 train_df['pred_lgb_ivle3'],test_df['pred_lgb_ivle3']=lgbmodelling(features,train_df,test_df)
 target = train_df['target']
features=['pred_lgb_ivgt3','pred_lgb_ivle3']
print(features)
 train_df['pred_lgb_iv3'],test_df['pred_lgb_iv3']=lgbmodelling(features,train_df,test_df )<save_to_csv>
test = pd.read_csv(f'{PATH}/test.csv')
 test.head()
sub_df = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df["target"] = test_df['pred_lgb_iv3']
sub_df.to_csv("lgbm pred_lgb_iv3.csv", index=False )<set_options>
x = X.fillna('NA' )
pd.set_option('display.max_columns', 200 )<load_from_csv>
X_train = train
X_train['Date'] = pd.to_datetime(X_train['Date'], infer_datetime_format=True)
 test['Date'] = pd.to_datetime(test['Date'], infer_datetime_format=True)
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<count_values>
EMPTY_VAL = "EMPTY_VAL"
def fillState(Province_State, Country_Region):
if Province_State == EMPTY_VAL: return Country_Region
return Province_State
train_df.target.value_counts()<init_hyperparams>
X_train_new = X_train.copy()
X_train_new['Province_State'].fillna(EMPTY_VAL, inplace=True)
 X_train_new['Province_State'] = X_train_new.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1)
 X_train_new.loc[:, 'Date'] = X_train_new.Date.dt.strftime("%m%d")
 X_train_new["Date"] = X_train_new["Date"].astype(int)
 X_train_new.head()
param = {
'num_leaves': 10,
'max_bin': 119,
'min_data_in_leaf': 11,
'learning_rate': 0.02,
'min_sum_hessian_in_leaf': 0.00245,
'bagging_fraction': 1.0,
'bagging_freq': 5,
'feature_fraction': 0.05,
'lambda_l1': 4.972,
'lambda_l2': 2.276,
'min_gain_to_split': 0.65,
'max_depth': 14,
'save_binary': True,
'seed': 1337,
'feature_fraction_seed': 1337,
'bagging_seed': 1337,
'drop_seed': 1337,
'data_random_seed': 1337,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': False,
}<define_variables>
l = preprocessing.LabelEncoder()
X_train_new.Country_Region = l.fit_transform(X_train_new.Country_Region)
 X_train_new['Province_State'] = l.fit_transform(X_train_new['Province_State'])
 X_train_new.head()
nfold = 5<define_variables>
X_Test = test.copy()
X_Test['Province_State'].fillna(EMPTY_VAL, inplace=True)
 X_Test['Province_State'] = X_Test.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1)
 X_Test.loc[:, 'Date'] = X_Test.Date.dt.strftime("%m%d")
 X_Test["Date"] = X_Test["Date"].astype(int)
 X_Test.head()
target = 'target'
predictors = train_df.columns.values.tolist() [2:]<set_options>
X_Test.Country_Region = l.fit_transform(X_Test.Country_Region)
 X_Test['Province_State'] = l.fit_transform(X_Test['Province_State'])
 X_Test.head()
gc.collect()<split>
p1_final = pd.DataFrame()
p3 = pd.DataFrame()
pred = pd.DataFrame()
p = list()
id_1 = pd.DataFrame()
id_2 = pd.DataFrame()
p1 =list()
for i in country:
id_1 = pd.concat([id_1,id_2])
 state = X_train_new.loc[X_train_new.Country_Region == i, :].Province_State.unique()
for j in state:
X_train_1 = X_train_new.loc[(X_train_new.Country_Region == i)&(X_train_new.Province_State == j), ['Province_State', 'Country_Region', 'Date', 'ConfirmedCases', 'Fatalities']]
Y_train1 = X_train_1.loc[:, 'ConfirmedCases']
Y_train2 = X_train_1.loc[:, 'Fatalities']
X_train_1 = X_train_1.loc[:, ['Province_State', 'Country_Region', 'Date']]
X_train_1.Country_Region = l.fit_transform(X_train_1.Country_Region)
 X_train_1['Province_State'] = l.fit_transform(X_train_1['Province_State'])
 X_test_1 = X_Test.loc[(X_Test.Country_Region == i)&(X_Test.Province_State == j), ['Province_State', 'Country_Region', 'Date', 'ForecastId']]
X_test_Id = X_test_1.loc[:, 'ForecastId']
X_test_1 = X_test_1.loc[:, ['Province_State', 'Country_Region', 'Date']]
X_test_1.Country_Region = l.fit_transform(X_test_1.Country_Region)
 X_test_1['Province_State'] = l.fit_transform(X_test_1['Province_State'])
 model_1 = XGBRegressor(n_estimators = 5000)
 model_1.fit(X_train_1, Y_train1)
 y_poly_pred_1 = model_1.predict(X_test_1)
 model_2 = XGBRegressor(n_estimators = 5000)
 model_2.fit(X_train_1, Y_train2)
 y_poly_pred_2 = model_2.predict(X_test_1)
 p3 = pd.DataFrame({'ForecastId': X_test_Id, 'ConfirmedCases': y_poly_pred_1, 'Fatalities': y_poly_pred_2})
 pred = pd.concat([pred,p3], axis=0)