Kaggle competition: COVID19 Global Forecasting (Week 2)
def process_test(clfs,
testdf,
full_train,
train_mean,
filename='submission.csv',
chunks=40615):
start = time.time()
chunks = 40615
testdf=testdf.round(5)
 testdf.to_csv(filename, index=False)
 for i_c, df in enumerate(pd.read_csv(filename, chunksize=chunks, iterator=True)) :
print(df.shape)
 preds_df = predict_chunk(df_=df,
clfs_=clfs,
features=full_train.columns,
train_mean=train_mean)
 if i_c == 0:
preds_df.to_csv('predictions.csv', header=True, mode='a', index=False)
 else:
preds_df.to_csv('predictions.csv', header=False, mode='a', index=False)
 del preds_df
gc.collect()
print('%15d done in %5.1f minutes' %(chunks *(i_c + 1),(time.time() - start)/ 60), flush=True)
 return<categorify>
plotly.offline.init_notebook_mode()
%matplotlib inline
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )
def GenUnknown(data):
return(((((( data["mymedian"])+(((data["mymean"])/ 2.0)))/2.0)) +(((((1.0)-(((data["mymax"])*(((data["mymax"])*(data["mymax"])))))))/ 2.0)))/2.0)
<init_hyperparams>
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )
best_params = {
'device': 'cpu',
'objective': 'multiclass',
'num_class': 14,
'boosting_type': 'gbdt',
'n_jobs': -1,
'max_depth': 5,
'n_estimators': 1000,
'subsample_freq': 2,
'subsample_for_bin': 5000,
'min_data_per_group': 100,
'max_cat_to_onehot': 4,
'cat_l2': 1.0,
'cat_smooth': 59.5,
'max_cat_threshold': 32,
'metric_freq': 10,
'verbosity': -1,
'metric': 'multi_logloss',
'xgboost_dart_mode': False,
'uniform_drop': False,
'colsample_bytree': 0.25,
'drop_rate': 0.173,
'learning_rate': 0.015,
'max_drop': 5,
'min_child_samples': 35,
'min_child_weight': 350.0,
'min_split_gain': 0.01,
'num_leaves': 31,
'reg_alpha': 0.01,
'reg_lambda': 0.000023,
'skip_drop': 0.44,
'subsample': 0.95} 
<load_from_csv>
FirstDate = train.groupby('Country_Region' ).min() ['Date'].unique() [0]
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 while train[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate)].shape[0] > 0:
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1)
 train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'ConfirmedCases'] = train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'Last Confirm']
train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Fatalities'] = train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Last Fatalities']
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1 )
trainingDartDf=pd.read_csv('.. /input/writefeaturetablefromsmotedartset/trainingFeatures1039.csv')
 trainingJimsDf=pd.read_csv('.. /input/normalizesomethingdifferentfeatures/traindfNormal.csv')
 if 'Unnamed: 0' in trainingDartDf.columns:
trainingDartDf=trainingDartDf.drop('Unnamed: 0', axis=1)
 print(trainingDartDf.shape)
 columnsToAdd=['outlierScore', 'hipd', 'lipd', 'highEnergy_transitory_1.0_TF',
'highEnergy_transitory_1.5_TF', 'lowEnergy_transitory_1.0_TF',
'lowEnergy_transitory_1.5_TF']
for column in columnsToAdd:
trainingDartDf.loc[:,column]=trainingJimsDf.loc[:,column]
traindf=trainingDartDf
del traindf['hostgal_specz']
del traindf['ra'], traindf['decl'], traindf['gal_l'], traindf['gal_b']
del traindf['ddf']
print(traindf.shape)
 traindf.head()<load_from_csv>
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
adjusted_X_pred = X_pred[feature_use].values.reshape(-1, 1)
 model = make_pipeline(PolynomialFeatures(2), BayesianRidge())
 model.fit(adjusted_X_train,adjusted_y_train_confirmed)
 y_hat_confirmed = model.predict(adjusted_X_pred)
 model.fit(adjusted_X_train,adjusted_y_train_fatalities)
 y_hat_fatalities = model.predict(adjusted_X_pred)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_1 = df_val.copy()
testDartDf=pd.read_csv('.. /input/writefeaturetablefromsmotedartset/feat_0.648970_2018-11-23-09-00.csv')
 testJimsDf=pd.read_csv('.. /input/normalizesomethingdifferentfeatures/testdfNormal.csv')
 if 'Unnamed: 0' in testDartDf.columns:
testDartDf=testDartDf.drop('Unnamed: 0', axis=1)
 print(testDartDf.shape)
 testDartDf.head()
for column in columnsToAdd:
testDartDf.loc[:,column]=testJimsDf.loc[:,column]
testdf=testDartDf
del testdf['hostgal_specz']
del testdf['ra'], testdf['decl'], testdf['gal_l'], testdf['gal_b']
del testdf['ddf']
testdf.shape<train_model>
RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values )
for cindex in full_train.columns:
full_train.loc[:,cindex]=np.float64(full_train.loc[:,cindex])
 eval_func = partial(lgbm_modeling_cross_validation,
full_train=full_train,
y=y,
classes=classes,
class_weights=class_weights,
nr_fold=12,
random_state=1)
 best_params.update({'n_estimators': 1000})
 clfs, score = eval_func(best_params )<load_from_csv>
RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values )
filename = 'subm_{:.6f}_{}.csv'.format(score,
dt.now().strftime('%Y-%m-%d-%H-%M'))
print('save to {}'.format(filename))
process_test(clfs,
testdf,
full_train,
train_mean=train_mean,
filename=filename,
chunks=40615)
 pdf = pd.read_csv('predictions.csv' )<prepare_x_and_y>
country = "Vietnam"
df_val = df_val_1
df_val[df_val['Country_Region'] == country].groupby(['Date','Country_Region'] ).sum().reset_index()
cols = list(pdf)
 cols.insert(0, cols.pop(cols.index('object_id')))
 pdf = pdf.loc[:, cols]
feats = ['class_6', 'class_15', 'class_16', 'class_42', 'class_52', 'class_53',
'class_62', 'class_64', 'class_65', 'class_67', 'class_88', 'class_90',
'class_92', 'class_95']
y = pd.DataFrame()
y['mymean'] = pdf[feats].mean(axis=1)
 y['mymedian'] = pdf[feats].median(axis=1)
 y['mymax'] = pdf[feats].max(axis=1)
 pdf['class_99'] = GenUnknown(y)
<merge>
animator.save('confirm_animation.gif', writer='imagemagick', fps=2)
 display(Image(url='confirm_animation.gif'))
def modUnknown(opdf, meta, ddfMult=0.5, mwMult=0.5, preserveMed=False):
pdf=copy.deepcopy(opdf)
 mdf=pdf.merge(meta,on='object_id')
 ddfilter=mdf.loc[:,'ddf']==1
mwfilter=mdf.loc[:,'hostgal_photoz']==0
print(ddfilter.sum())
 print(mwfilter.sum())
 mdf.loc[mwfilter,'class_99']=mwMult*mdf.loc[mwfilter,'class_99']
mdf.loc[ddfilter,'class_99']=ddfMult*mdf.loc[ddfilter,'class_99']
pdf.loc[:,'class_99']=mdf.loc[:,'class_99']
return pdf
npdf=modUnknown(pdf, meta)
 npdf.head() 
<save_to_csv>
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = ExponentialSmoothing(adjusted_y_train_confirmed, trend = 'additive' ).fit()
y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = ExponentialSmoothing(adjusted_y_train_fatalities, trend = 'additive' ).fit()
y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_2 = df_val.copy()
npdf.to_csv(filename, index=False)
<categorify>
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()
def testizeTrain(traindf, dupMult=5):
for classval in traindf.loc[:,'target'].unique() :
classfilter=traindf.loc[:,'target']==classval
count=classfilter.sum()
if np.max(traindf.loc[classfilter,'distmod'])>0:
todup=np.ceil(count/4 ).astype(int)
 dups=traindf.loc[classfilter,:].nlargest(todup,'distmod')
 for i in range(dupMult):
traindf=traindf.append(dups,ignore_index=True)
 return traindf
def testizeTrainSmoteNoRep(traindf):
for classval in traindf.loc[:,'target'].unique() :
classfilter=traindf.loc[:,'target']==classval
count=classfilter.sum()
if np.max(traindf.loc[classfilter,'distmod'])>0:
todup=np.ceil(count/4 ).astype(int)
 dups=traindf.loc[classfilter,:].nlargest(todup,'distmod')
 mindm=np.min(dups.loc[:,'distmod'])
 flagfilter=(traindf.loc[:,'distmod']>=mindm)& classfilter
traindf.loc[flagfilter,'target']+=100
return traindf<load_from_csv>
method_list = ['Poly Bayesian Ridge','Exponential Smoothing','SARIMA']
method_val = [df_val_1,df_val_2,df_val_3]
for i in range(0,3):
df_val = method_val[i]
method_score = [method_list[i]] + [RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values)] + [RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values)]
print(method_score )
<load_from_csv>
df_val = df_val_1
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission.to_csv('submission.csv', index=False)
 submission
<feature_engineering><EOS> 
df_worldinfor[df_worldinfor['Country'] == 'Vietnam']