Kaggle competition: COVID19 Global Forecasting (Week 1)
LGB_PARAMS = {
'objective': 'regression',
'metric': 'mae',
'verbosity': -1,
'boosting_type': 'gbdt',
'learning_rate': 0.2,
'num_leaves': 128,
'min_child_samples': 79,
'max_depth': 9,
'subsample_freq': 1,
'subsample': 0.9,
'bagging_seed': 11,
'reg_alpha': 0.1,
'reg_lambda': 0.3,
'colsample_bytree': 1.0
}<predict_on_test>
train = pd.read_csv('.. /input/covid19-global-forecasting-week-1/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-1/test.csv')
 submission = pd.read_csv('.. /input/covid19-global-forecasting-week-1/submission.csv')
model = LGBMRegressor(**LGB_PARAMS, n_estimators=1500, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_val, y_val)], eval_metric='mae',
verbose=100, early_stopping_rounds=200)
 y_pred = model.predict(X_val)
 np.log(mean_absolute_error(y_val, y_pred))<prepare_x_and_y>
test['Date']=test.Date.astype('datetime64[ns]' )
def build_x_y_data(some_csv, coupling_type, n_atoms):
full = build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=n_atoms)
 df = take_n_atoms(full, n_atoms)
 df = df.fillna(0)
 print(df.columns)
 if 'scalar_coupling_constant' in df:
X_data = df.drop(['scalar_coupling_constant'], axis=1 ).values.astype('float32')
 y_data = df['scalar_coupling_constant'].values.astype('float32')
 else:
X_data = df.values.astype('float32')
 y_data = None
return X_data, y_data<find_best_model_class>
train['key']=train['Province/State'].astype('str')+ " " + train['Country/Region'].astype('str')+ " " +train['Lat'].astype('str')+ " " +train['Long'].astype('str')
 test['key']=test['Province/State'].astype('str')+ " " + test['Country/Region'].astype('str')+ " " +test['Lat'].astype('str')+ " " +test['Long'].astype('str')
 train.describe()
def train_and_predict_for_one_coupling_type(coupling_type, submission, n_atoms, n_folds=5, n_splits=5, random_state=128):
print(f'*** Training Model for {coupling_type} ***')
 X_data, y_data = build_x_y_data(train_csv, coupling_type, n_atoms)
 X_test, _ = build_x_y_data(test_csv, coupling_type, n_atoms)
 y_pred = np.zeros(X_test.shape[0], dtype='float32')
 cv_score = 0
if n_folds > n_splits:
n_splits = n_folds
kfold = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)
 for fold,(train_index, val_index)in enumerate(kfold.split(X_data, y_data)) :
if fold >= n_folds:
break
X_train, X_val = X_data[train_index], X_data[val_index]
y_train, y_val = y_data[train_index], y_data[val_index]
model = LGBMRegressor(**LGB_PARAMS, n_estimators=1500, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_val, y_val)], eval_metric='mae',
verbose=100, early_stopping_rounds=200)
 y_val_pred = model.predict(X_val)
 val_score = np.log(mean_absolute_error(y_val, y_val_pred))
print(f'{coupling_type} Fold {fold}, logMAE: {val_score}')
 cv_score += val_score / n_folds
y_pred += model.predict(X_test)/ n_folds
submission.loc[test_csv['type'] == coupling_type, 'scalar_coupling_constant'] = y_pred
return cv_score<find_best_params>
daily_analysis=train.groupby(['Date'] ).sum()
model_params = {
'1JHN': 7,
'1JHC': 10,
'2JHH': 9,
'2JHN': 9,
'2JHC': 9,
'3JHH': 9,
'3JHC': 10,
'3JHN': 10
}
N_FOLDS = 3
submission = submission_csv.copy()
cv_scores = {}
for coupling_type in model_params.keys() :
cv_score = train_and_predict_for_one_coupling_type(
 coupling_type, submission, n_atoms=model_params[coupling_type], n_folds=N_FOLDS)
 cv_scores[coupling_type] = cv_score<save_to_csv>
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )
submission.to_csv(f'{SUBMISSIONS_PATH}/submission.csv' )<load_from_csv>
train_lag_1=train.groupby(['key'] ).shift(periods=1)
 train_lag_2=train.groupby(['key'] ).shift(periods=2)
 train_lag_3=train.groupby(['key'] ).shift(periods=3)
 train['lag_1_ConfirmedCases']=train_lag_1['ConfirmedCases']
train['lag_1_Fatalities']=train_lag_1['Fatalities']
train['lag_2_ConfirmedCases']=train_lag_2['ConfirmedCases']
train['lag_2_Fatalities']=train_lag_2['Fatalities']
train['lag_3_ConfirmedCases']=train_lag_3['ConfirmedCases']
train['lag_3_Fatalities']=train_lag_3['Fatalities']
train
sub1 = pd.read_csv('.. /input/mysubmissions/submission(-1.581 ).csv')
 sample = pd.read_csv('.. /input/champs-scalar-coupling/sample_submission.csv')
 sub2 = pd.read_csv('.. /input/mysubmissions/submission(-1.587 ).csv' )<save_to_csv>
orig_stdout = sys.stdout
Fatalities_all_result_final=pd.DataFrame()
ConfirmedCases_all_result_Final=pd.DataFrame()
for keys in train['key'].unique() :
chk=train[train['key']==keys]
chk.index=chk.Date
fcastperiod=0
fcastperiod1=35
actual=chk[:chk.shape[0]-fcastperiod]
ffcast=chk[chk.shape[0]-fcastperiod-1:]
ffcast
try:
Fatalities_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'Fatalities' ).reset_index()
except:
Fatalities_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 Fatalities_all_result_1.columns=['index']
Fatalities_all_result_1['best_model']='naive'
Fatalities_all_result_1['best_pred']=0
Fatalities_all_result_1['key']=keys
Fatalities_all_result_final=Fatalities_all_result_final.append(Fatalities_all_result_1,sort=True)
 try:
ConfirmedCases_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'ConfirmedCases' ).reset_index()
except:
ConfirmedCases_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 ConfirmedCases_all_result_1.columns=['index']
ConfirmedCases_all_result_1['best_model']='naive'
ConfirmedCases_all_result_1['best_pred']=1
ConfirmedCases_all_result_1['key']=keys
ConfirmedCases_all_result_Final=ConfirmedCases_all_result_Final.append(ConfirmedCases_all_result_1,sort=True)
 print(' done for %s' % keys)
 sys.stdout = orig_stdout
sample['scalar_coupling_constant'] = sub2['scalar_coupling_constant']
sample.to_csv('stackers_blend.csv', index=False )<define_variables>
ConfirmedCases_all_result_Final.rename(columns={'index':'Date'},inplace=True)
 Fatalities_all_result_final.rename(columns={'index':'Date'},inplace=True)
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['AdditiveDamped'] is np.nan , ConfirmedCases_all_result_Final['best_pred'] ,
ConfirmedCases_all_result_Final['AdditiveDamped'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['AdditiveDamped'] is np.nan , Fatalities_all_result_final['best_pred'] ,
Fatalities_all_result_final['AdditiveDamped'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 ,
Fatalities_all_result_final['best_pred'] )
sub_path = ".. /input/top-mol"
all_files = os.listdir(sub_path)
 all_files<load_from_csv>
test['Date']=test.Date.astype('datetime64[ns]' )
outs = [pd.read_csv(os.path.join(sub_path, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "mol" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head()
ncol = concat_sub.shape[1]<feature_engineering>
eval1 = ConfirmedCases_all_result_Final[['key','Date','best_pred','best_pred_1']].merge(test, how='right', on=['key','Date'])
 eval1.rename(columns={'best_pred':'ConfirmedCases'},inplace=True)
 eval1['ConfirmedCases']=eval1['ConfirmedCases'].fillna(0)
 eval1
concat_sub['m_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)
 concat_sub['m_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)
 concat_sub['m_mean'] = concat_sub.iloc[:, 1:ncol].mean(axis=1)
 concat_sub['m_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1 )<define_variables>
eval2 = Fatalities_all_result_final[['key','Date','best_pred','best_pred_1']].merge(test, how='right', on=['key','Date'])
 eval2.rename(columns={'best_pred':'Fatalities'},inplace=True)
 eval2['Fatalities']=eval2['Fatalities'].fillna(0)
 eval2
cutoff_lo = 0.8
cutoff_hi = 0.2<save_to_csv>
sub_prep = eval1[['ForecastId','ConfirmedCases','key']].merge(eval2[['ForecastId','Fatalities']], on=['ForecastId'], how='left')
 sub_prep
concat_sub['scalar_coupling_constant'] = concat_sub['m_mean']
concat_sub[['id', 'scalar_coupling_constant']].to_csv('stack_mean.csv',
index=False, float_format='%.6f' )<save_to_csv>
sub = sub_prep.merge(submission['ForecastId'], on=['ForecastId'], how='right')
 sub
concat_sub['scalar_coupling_constant'] = concat_sub['m_median']
concat_sub[['id', 'scalar_coupling_constant']].to_csv('stack_median.csv',
index=False, float_format='%.6f' )<save_to_csv>
sub=sub[['ForecastId','ConfirmedCases','Fatalities']]
sub=sub.sort_values('ForecastId')
 sub
concat_sub['scalar_coupling_constant'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), 1,
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
0, concat_sub['m_median']))
concat_sub[['id', 'scalar_coupling_constant']].to_csv('stack_pushout_median.csv',
index=False, float_format='%.6f' )<feature_engineering>
sub.to_csv('submission.csv',header=['ForecastId','ConfirmedCases','Fatalities'],index=False )
concat_sub['scalar_coupling_constant'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['m_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['m_min'],
concat_sub['m_mean']))
concat_sub[['id', 'scalar_coupling_constant']].to_csv('stack_minmax_mean.csv',
index=False, float_format='%.6f' )<feature_engineering>
train['Date']=train.Date.astype('datetime64[ns]')
 verify=train[['key','Date','ConfirmedCases','Fatalities']].merge(test[['key','Date','ForecastId']], how='inner', on=['key','Date'])
 pred=verify[['ForecastId']].merge(sub, how='inner', on=['ForecastId'] )
concat_sub['scalar_coupling_constant'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['m_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['m_min'],
concat_sub['m_median']))
concat_sub[['id', 'scalar_coupling_constant']].to_csv('stack_minmax_median.csv',
index=False, float_format='%.6f' )<save_to_csv>
RMSLE(pred['Fatalities'].values,verify['Fatalities'].values )
concat_sub['scalar_coupling_constant'] = concat_sub['mol0'].rank(method ='min')+ concat_sub['mol1'].rank(method ='min')+ concat_sub['mol2'].rank(method ='min')
 concat_sub['scalar_coupling_constant'] =(concat_sub['scalar_coupling_constant']-concat_sub['scalar_coupling_constant'].min())/(concat_sub['scalar_coupling_constant'].max() - concat_sub['scalar_coupling_constant'].min())
 concat_sub.describe()
concat_sub[['id', 'scalar_coupling_constant']].to_csv('stack_rank.csv', index=False, float_format='%.8f' )<load_from_csv>
RMSLE(pred['ConfirmedCases'].values,verify['ConfirmedCases'].values )
sub1=pd.read_csv('.. /input/another-one/stackers_blend.csv')
 sub2=pd.read_csv('.. /input/lgb-public-kernels-plus-more-features/sub_lgb_model_individual.csv')
 sub3=pd.read_csv('.. /input/yet-another-one/stackers_blend.csv')
 sub4=pd.read_csv('.. /input/giba-r-data-table-simple-features-1-17-lb/submission-giba-1.csv')
 temp=pd.read_csv('.. /input/another-one/stackers_blend.csv' )<save_to_csv>
best_model_key=ConfirmedCases_all_result_Final[['key','best_model']].drop_duplicates()
max_number_current=train.groupby('key' ).max() [['ConfirmedCases','Fatalities']].reset_index()
best_model_key=best_model_key.merge(max_number_current,on='key',how='left')
 best_model_key=best_model_key.sort_values('ConfirmedCases',ascending=False ).reset_index(drop=True )
temp['scalar_coupling_constant'] = 0.6*sub1['scalar_coupling_constant'] + 0.4*sub1['scalar_coupling_constant']
temp.to_csv('submission4.csv', index=False )<load_from_csv>
train_ck=train.groupby(['Country/Region','Date'] ).sum().reset_index()
train_ck['key']=train_ck['Country/Region']
sub4 = pd.read_csv('.. /input/from-sub-lgb/sub_lgb_model_individual.csv')
 sub6 = pd.read_csv('.. /input/stealing/submission.csv')
 sub7 = pd.read_csv('.. /input/stealing/submission1.csv')
 print(sub4['scalar_coupling_constant'].describe())
 print(sub6['scalar_coupling_constant'].describe())
 print(sub7['scalar_coupling_constant'].describe())
<save_to_csv>
Fatalities_all_result_final=pd.DataFrame()
ConfirmedCases_all_result_Final=pd.DataFrame()
for keys in train_ck['key'].unique() :
chk=train_ck[train_ck['key']==keys]
chk.index=chk.Date
fcastperiod=0
fcastperiod1=35
actual=chk[:chk.shape[0]-fcastperiod]
ffcast=chk[chk.shape[0]-fcastperiod-1:]
ffcast
try:
Fatalities_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'Fatalities' ).reset_index()
except:
Fatalities_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 Fatalities_all_result_1.columns=['index']
Fatalities_all_result_1['best_model']='naive'
Fatalities_all_result_1['best_pred']=0
Fatalities_all_result_1['key']=keys
Fatalities_all_result_final=Fatalities_all_result_final.append(Fatalities_all_result_1,sort=True)
 try:
ConfirmedCases_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'ConfirmedCases' ).reset_index()
except:
ConfirmedCases_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 ConfirmedCases_all_result_1.columns=['index']
ConfirmedCases_all_result_1['best_model']='naive'
ConfirmedCases_all_result_1['best_pred']=1
ConfirmedCases_all_result_1['key']=keys
ConfirmedCases_all_result_Final=ConfirmedCases_all_result_Final.append(ConfirmedCases_all_result_1,sort=True)
 print(' done for %s' % keys )
sub6['scalar_coupling_constant'] = 0.34*sub6['scalar_coupling_constant'] + 0.33*sub7['scalar_coupling_constant'] + 0.33*sub4['scalar_coupling_constant']
sub6.to_csv('submission.csv', index=False )<save_to_csv>
ConfirmedCases_all_result_Final.rename(columns={'index':'Date'},inplace=True)
 Fatalities_all_result_final.rename(columns={'index':'Date'},inplace=True)
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['AdditiveDamped'] is np.nan , ConfirmedCases_all_result_Final['best_pred'] ,
ConfirmedCases_all_result_Final['AdditiveDamped'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['AdditiveDamped'] is np.nan , Fatalities_all_result_final['best_pred'] ,
Fatalities_all_result_final['AdditiveDamped'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 ,
Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0,
ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 ,
Fatalities_all_result_final['best_pred'] )
<define_variables><EOS> 
best_model_key=ConfirmedCases_all_result_Final[['key','best_model']].drop_duplicates()
max_number_current=train_ck.groupby('key' ).max() [['ConfirmedCases','Fatalities']].reset_index()
best_model_key=best_model_key.merge(max_number_current,on='key',how='left')
 best_model_key=best_model_key.sort_values('ConfirmedCases',ascending=False ).reset_index(drop=True )