Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<train_model>
pio.templates.default = "plotly_dark"
oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(len(X_test))
feats = [f for f in train_df.columns]
for n_fold,(train_idx, valid_idx)in enumerate(skf_three.split(train_df[feats], label_df)) :
trn_data = lgb.Dataset(train_df.iloc[train_idx][feats], label=label_df.iloc[train_idx])
 val_data = lgb.Dataset(train_df.iloc[valid_idx][feats], label=label_df.iloc[valid_idx])
 clf = lgb.train(param, trn_data,1000000, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 3500)
 oof_preds[valid_idx] = clf.predict(train_df.iloc[valid_idx][feats], num_iteration=clf.best_iteration)
 sub_preds += clf.predict(X_test[feats], num_iteration=clf.best_iteration)/ 5
print('Full AUC score %.6f' % roc_auc_score(label_df, oof_preds))
pred3=sub_preds<save_to_csv>
train_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )
santander_submission['target'] = pred3
santander_submission.to_csv('submission23.csv', index=False )<set_options>
train_df.isna().sum()
init_notebook_mode()
warnings.filterwarnings('ignore' )<init_hyperparams>
test_df.isna().sum()
sampledf = False
frac_sample = 0.01
augmnt = False
fold_train = True
agmnt_between = True
kfold_shuffle = False
use_perc = False
log_transf = False
sq_data = True
dim_red = False
n_components = 60
standradize=False
train_between = True
freq = True
feature_eng_cv = True
weighted = False
balanced = False
tst_size = 0.3
sub_name = 'submission'
print('Options Active:
\t SampledDF: {} frac: {}
\t Augmentation: {}
\t Weighted: {}
\t Balanced: {}\
\t agmnt_between:{}
\t Percentiles: {}
\t LOG_Transform: {}
\t PCA: {}
\t Square: {}
\t Standradize: {}
\t Train_Between: {}
\t Freq: {}'.format(sampledf,frac_sample,augmnt,
weighted,balanced,agmnt_between,use_perc,log_transf
,dim_red,sq_data,standradize,train_between,freq))<load_from_csv>
all_data = pd.concat([train_df,test_df],axis=0,sort=False)
 all_data['Province_State'].fillna("None", inplace=True)
 all_data['ConfirmedCases'].fillna(0, inplace=True)
 all_data['Fatalities'].fillna(0, inplace=True)
 all_data['Id'].fillna(-1, inplace=True)
 all_data['ForecastId'].fillna(-1, inplace=True)
df_t = pd.read_csv('.. /input/train.csv')
 df_tst = pd.read_csv('.. /input/test.csv')
 if sampledf:
sub_name = sub_name+'_sampled'
df_train = df_t.sample(frac=(frac_sample))
df_test = df_tst.sample(frac=(frac_sample))
print('Loading Sampled df.. ')
 else:
df_train = df_t.copy()
df_test = df_tst.copy()
print('Training df shape',df_train.shape)
 print('Test df shape',df_test.shape )<define_variables>
le = LabelEncoder()
all_data['Date'] = pd.to_datetime(all_data['Date'])
 all_data['Day_num'] = le.fit_transform(all_data.Date)
 all_data['Day'] = all_data['Date'].dt.day
all_data['Month'] = all_data['Date'].dt.month
all_data['Year'] = all_data['Date'].dt.year
df_ones = df_train[df_train['target'] > 0]
print('Ones',df_ones.shape)
 df_zeros = df_train[df_train['target'] == 0].sample(frac=0.25)
 print('Zeros',df_zeros.shape)
 df_sampling = pd.concat([df_ones, df_zeros] ).sample(frac=1)
 print(df_sampling.shape )<categorify>
train = all_data[all_data['ForecastId']==-1.0]
test = all_data[all_data['ForecastId']!=-1.0]
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<feature_engineering>
train['Province_State'] = le.fit_transform(train['Province_State'])
 train['Country_Region'] = le.fit_transform(train['Country_Region'])
 test['Province_State'] = le.fit_transform(test['Province_State'])
 test['Country_Region'] = le.fit_transform(test['Country_Region'])
 X = train.drop(columns=['Id','ConfirmedCases','Fatalities','Date','ForecastId'],axis=1)
 cases = train.ConfirmedCases
fatalities = train.Fatalities
x_test = test.drop(columns=['Id','ConfirmedCases','Fatalities','Date','ForecastId'],axis=1 )
def eng_features_preprocess(df,idx,use_perc,freq,log_transf,sq_data,perc_list =[1,2,5,10,25,50,60,75,80,85,95,99]):
print(' * Loading new data metrics: ')
 df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1)
 print(' * Loading moving average metric: ')
 df['ma'] = df[idx].apply(lambda x: np.ma.average(x), axis=1)
 if standradize:
print(' * Standradizing the data.. ')
 scaler = StandardScaler()
df_train.iloc[:,2:] = scaler.fit_transform(df_train.iloc[:,2:])
 df_test.iloc[:,1:] = scaler.fit_transform(df_test.iloc[:,1:])
 print(' * Data Standradized!')
 if use_perc:
print(' * Loading percentiles: ')
 for i in perc_list:
df['perc_'+str(i)] = df[idx].apply(lambda x: np.percentile(x, i), axis=1)
 if freq:
print(' * Loading frequency: ')
 for var in idx:
hist, bin_edges = np.histogram(df[var], bins=1000, density=True)
 hist_test, bin_edges_test = np.histogram(df[var], bins=1000, density=True)
 df['test_'+var] = [ hist[np.searchsorted(bin_edges,ele)-1] for ele in df[var]]
if log_transf:
print(' * Loading log transformations')
 for i in range(200):
df['var_log_'+str(i)] = np.log(df['var_'+str(i)])
 if sq_data:
print(' * Loading Squared data: ')
 for i in range(200):
df['var_sq_'+str(i)] = np.square(df['var_'+str(i)] )<feature_engineering>
model = XGBRegressor(n_estimators = 1000 , random_state = 0 , max_depth = 15)
 model.fit(X,cases)
 cases_pred = model.predict(x_test)
 model1 = XGBRegressor(n_estimators = 1000 , random_state = 0 , max_depth = 15)
 model1.fit(X,fatalities)
 fatalities_pred = model1.predict(x_test)
if not feature_eng_cv:
def feature_creation(df,idx,use_perc,perc_list,freq,log_transf,name_num='_1'):
print(' * Loading new data metrics: ', name_num)
 df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1)
 print(' * Loading moving average metric: ', name_num)
 df['ma'] = df[idx].apply(lambda x: np.ma.average(x), axis=1)
 if use_perc:
print(' * Loading percentiles: ', name_num)
 for i in perc_list:
df['perc_'+str(i)] = df[idx].apply(lambda x: np.percentile(x, i), axis=1)
 perc_size = 0
perc_list = [1,2,5,10,25,50,60,75,80,85,95,99]
if use_perc:
perc_size = len(perc_list)
 start_time = time.time()
for i,df in enumerate([df_train,df_test]):
print('Loading more features for df: {}/{}'.format(i+1,3))
print('Creating Metrics Part 1')
 features_1 = df_train.columns.values[2:202]
feature_creation(df,features_1,use_perc,perc_list,name_num='_1')
 print('Creating Metrics Part 2')
 features_2 = df_train.columns.values[2:211+perc_size]
feature_creation(df,features_2,use_perc,perc_list,name_num='_2')
 df.drop(['min_2','max_2'],axis=1,inplace=True)
 print('-'*50)
 if freq:
print('Creating freq.. ')
 for var in df_train.columns.values[2:202]:
hist, bin_edges = np.histogram(df_train[var], bins=1000, density=True)
 hist_test, bin_edges_test = np.histogram(df_train[var], bins=1000, density=True)
 df_train['test_'+var] = [ hist[np.searchsorted(bin_edges,ele)-1] for ele in df_train[var] ]
df_test['test_'+var] = [ hist_test[np.searchsorted(bin_edges_test,ele)-1] for ele in df_test[var] ]
if log_transf:
print('Loading log transformations')
 for i in range(200):
df_train['var_log_'+str(i)] = np.log(df_train['var_'+str(i)])
 df_test['var_log_'+str(i)] = np.log(df_test['var_'+str(i)])
 if sq_data:
print('Loading Squared data.. ')
 for i in range(200):
df_train['var_sq_'+str(i)] = np.square(df_train['var_'+str(i)])
 df_test['var_sq_'+str(i)] = np.square(df_test['var_'+str(i)])
 print('Features loaded !')
 print("Execution --- %s seconds ---" %(time.time() - start_time))
print('Train df: ', df_train.columns)
 print('Test df: ', df_test.columns)
 print('Number of Features: ', len(df_train.columns[2:]))<prepare_x_and_y>
MSE = mean_squared_error(cases.iloc[0:13459],cases_pred)
 RMSE = sqrt(mean_squared_error(cases.iloc[0:13459],cases_pred))
MAE = mean_absolute_error(cases.iloc[0:13459],cases_pred)
 R2 = r2_score(cases.iloc[0:13459],cases_pred)
 print('Mean squared error :', MSE)
 print('Root mean squared error :',RMSE)
 print('Mean absolute error :', MAE)
 print('R squared :',R2 )
X = df_train.iloc[:,2:]
Y = df_train['target']
X_target = df_test.iloc[:,1:]
if augmnt:
print('Data Augmentation: Enabled')
 X,Y = augment(X.values,Y.values,t=2)
 X = pd.DataFrame(X,columns=df_train.columns[2:])
 Y = pd.Series(Y)
 print('Augmentation Succeeded')
 labels = ["0","1"]
values = [(sum(Y == 0)/len(Y)) *100,(sum(Y > 0)/len(Y)) *100]
vis_classes(labels,values,title ='Target Percentages After Augmentation')
 sub_name = sub_name+'_agmted'<categorify>
test_df_predict = test_df.copy()
test_df_predict['Confirmedcase'] = cases_pred
test_df_predict['Fatalities'] = fatalities_pred
test_df_predict = test_df_predict.drop('Province_State',axis=1)
 test_df_predict.to_csv('Forecast_result.csv' )
if dim_red :
print('Reducing Dimension to:',n_components)
 pca = PCA(n_components= n_components)
 xnew =pca.fit_transform(X)
 xtest = pca.fit_transform(X_target)
 X =pd.DataFrame(xnew,columns=['pc_'+str(i)for i in range(n_components)])
 X_target = pd.DataFrame(xtest,columns=['pc_'+str(i)for i in range(n_components)])
 print(X.shape)
 X.head()<split>
US = test_df_predict[test_df_predict['Country_Region']=='US']
US.groupby('Date')['Confirmedcase','Fatalities'].sum().reset_index()
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=tst_size, random_state=6666)
 X_smple = df_sampling.iloc[:,2:]
y_smple = df_sampling['target']
X_train_smple, X_test_smple, y_train_smple, y_test_smple = train_test_split(X_smple, y_smple, test_size=0.4, random_state=6)
 print("X_train: ", X_train.shape)
 print("X_test: " ,X_test.shape )<init_hyperparams>
test_df_predict.groupby(['Date','Country_Region'])['Confirmedcase','Fatalities'].max().reset_index().head(10)
param = {
'bagging_freq': 10,
'bagging_fraction': 0.2,
'boost_from_average':False,
'boost': 'gbdt',
'feature_fraction': 0.15,
'learning_rate': 0.01,
'max_depth':2,
'metric':'auc',
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'xentropy',
'verbosity':1,
"bagging_seed" : 122,
"seed": 20,
}
def create_model_lgbm(param,X_train,y_train,X_val=None,y_val=None):
dtrain = lgb.Dataset(X_train,label=y_train)
 if not X_val is None:
dval = lgb.Dataset(X_val,label=y_val)
 valid_sets =(dtrain,dval)
 valid_names = ['train','valid']
num_boost_round = 200000
else:
valid_sets =(dtrain)
 valid_names = ['train']
num_boost_round = 60000
model = lgb.train(param,dtrain,num_boost_round=num_boost_round,valid_sets=valid_sets,valid_names=valid_names,
verbose_eval=3000,
early_stopping_rounds=3000)
 return model
print('Processing and FE Test Data.. ')
 eng_features_preprocess(X_target,X_target.columns,use_perc,freq,log_transf,sq_data,perc_list =[1,2,5,10,25,50,60,75,80,85,95,99])
 lgbm_test_x = X_target
predictions = df_test[['ID_code']]
print('Using Params:
',param)
 val_aucs = []
val_pred = 0
target_pred = 0
importand_folds = 0
sub_train_n =2
kf = StratifiedKFold(n_splits=5,shuffle = kfold_shuffle, random_state=546)
 if fold_train:
for _fold,(trn_idx, val_idx)in enumerate(kf.split(X.values, Y.values)) :
Xtrn, ytrn = X.iloc[trn_idx], Y.iloc[trn_idx]
Xval, y_val = X.iloc[val_idx], Y.iloc[val_idx]
if feature_eng_cv:
print('FE on training data.. ')
 eng_features_preprocess(Xtrn,Xtrn.columns,use_perc,freq,log_transf,sq_data,perc_list =[1,2,5,10,25,50,60,75,80,85,95,99])
 print('FE on val data.. ')
 eng_features_preprocess(Xval,Xval.columns,use_perc,freq,log_transf,sq_data,perc_list =[1,2,5,10,25,50,60,75,80,85,95,99])
 print('Num of features : ', len(Xtrn.columns))
ones_train =(sum(ytrn>0)/ len(ytrn)) *100
ones_val =(sum(y_val>0)/ len(y_val)) *100
print('-'*50)
 print("Fold num:{}".format(_fold + 1))
print('\tTrain Perc: 1: {:.2f}%, 0: {:.2f}%'.format(ones_train,100-ones_train))
print('\tValid Perc: 1 : {:.2f}%, 0:{:.2f}%'.format(ones_val,100-ones_val))
if train_between:
val_pred = 0
target_pred = 0
for i in range(sub_train_n):
print('\tSub-Train: {}'.format(i+1))
if agmnt_between:
X_t, y_t = augment(Xtrn.values, ytrn.values)
 print('\tAugmentation Succeeded.. ')
 X_t = pd.DataFrame(X_t)
 X_t = X_t.add_prefix('var_')
 print('\tFitting Model')
 clf = create_model_lgbm(param,X_t,y_t,Xval,y_val)
 else:
clf = create_model_lgbm(param,Xtrn,ytrn,Xval,y_val)
 target_pred += clf.predict(lgbm_test_x)
 val_pred += clf.predict(Xval)
 else:
clf = create_model_lgbm(param,Xtrn,ytrn,Xval,y_val)
 val_pred = clf.predict(Xval)
 target_pred = clf.predict(X_target)
 print('-' * 50)
 importand_folds += clf.feature_importance()
if train_between:
val_score = roc_auc_score(y_val, val_pred/sub_train_n)
 predictions['fold{}'.format(_fold+1)] = target_pred/sub_train_n
else:
val_score = roc_auc_score(y_val, val_pred)
 predictions['fold{}'.format(_fold+1)] = target_pred
val_aucs.append(val_score)
 print('\tVal CV score : {}'.format(val_score))
print('-' * 50)
 mean_cv_score = np.mean(val_aucs)
 print('----- Mean CV Score: {:.2} ------'.format(mean_cv_score))<find_best_params>
test_df_predict.groupby('Country_Region')['Confirmedcase', 'Fatalities'].sum().reset_index().sort_values(by='Confirmedcase',ascending=False ).head(15)
<compute_test_metric><EOS> 
cases_pred = [round(value)for value in cases_pred ]
fatalities_pred = [round(value)for value in fatalities_pred ]
submission['ConfirmedCases'] = cases_pred
submission['Fatalities'] = fatalities_pred
submission.to_csv('submission.csv',index=False)