Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<init_hyperparams>
sns.set_style('darkgrid')
 %matplotlib inline
warnings.filterwarnings('ignore')
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 350,
'max_bin':700,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.58,
'bagging_fraction': 0.58,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.05,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df.sample(n=20000*1000)
 print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.025)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 valid_df=train_df_clf.sample(frac=0.09)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('valid_df length：',len(valid_df)) 
<drop_column>
df_train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 df_train['Date'] = pd.to_datetime(df_train['Date'], format = '%Y-%m-%d')
 df_test['Date'] = pd.to_datetime(df_test['Date'], format = '%Y-%m-%d')
 stamp = dt.datetime(2020, 4, 14)
 df_train = df_train[df_train['Date']<=stamp]
df_test = df_test[df_test['Date']<=stamp]
df_train['Days'] = df_train['Date']-df_train['Date'].min()
df_train.Province_State.fillna('ZZZ', inplace=True)
 df_test.Province_State.fillna('ZZZ', inplace=True)
 df_train['Days'] = df_train['Days'].astype('timedelta64[D]')
del train_df_clf
del valid_df
gc.collect()<prepare_x_and_y>
dict_geo = {}
df_mid = df_train.groupby(['Country_Region','Province_State'],as_index=False)
 print(len(df_mid))
df_metadata = pd.read_csv('/kaggle/input/covid19-forecasting-metadata/region_metadata.csv')
 df_metadata.Province_State.fillna('ZZZ', inplace=True)
 dict_key = df_metadata.groupby(['Country_Region','Province_State'],as_index=False)
 for name,group in df_mid:
i = df_metadata[(df_metadata['Country_Region']== name[0])&(df_metadata['Province_State']== name[1])].index.tolist() [0]
lat = df_metadata.loc[i,'lat']
long = df_metadata.loc[i,'lon']
pop = df_metadata.loc[i,'population']
area = df_metadata.loc[i,'area']
dict_geo.update({name:[lat,long,pop,area]} )
for i in range(0,num):
tr_data = lgb.Dataset(trains[i][features], label=trains[i][target])
 va_data = lgb.Dataset(valids[i][features], label=valids[i][target])
 del trains
del valids
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=5000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 lgb.plot_importance(model, importance_type='gain')
 plt.show()
del tr_data
del va_data
gc.collect() 
<data_type_conversions>
def add_daily_measures(df):
df.loc[0,'Daily Cases'] = df.loc[0,'ConfirmedCases']
df.loc[0,'Daily Deaths'] = df.loc[0,'Fatalities']
for i in range(1,len(df)) :
df.loc[i,'Daily Cases'] = df.loc[i,'ConfirmedCases'] - df.loc[i-1,'ConfirmedCases']
df.loc[i,'Daily Deaths'] = df.loc[i,'Fatalities'] - df.loc[i-1,'Fatalities']
if df.loc[i,'Daily Deaths']<0:
df.loc[i,'Daily Deaths'] = df.loc[i-1,'Daily Deaths']
if df.loc[i,'Daily Cases']<0:
df.loc[i,'Daily Cases'] = df.loc[i-1,'Daily Cases']
df.loc[i,'lastconfirm'] = df.loc[i-1,'ConfirmedCases']
df.loc[i,'lastdeath'] = df.loc[i-1,'Fatalities']
df.loc[0,'Daily Cases'] = 0
df.loc[0,'Daily Deaths'] = 0
df.loc[0,'lastconfirm'] = 0
df.loc[0,'lastdeath'] = 0
return df
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))
del user_agg
del content_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()<data_type_conversions>
def train_dev_split(df, days):
date = df['Date'].max() - dt.timedelta(days=days)
 return df,df[df['Date'] <= date], df[df['Date'] > date]
def add_more_daily_measures(df):
for i in range(4,len(df)) :
df.loc[i,'Daily Cases 1'] = df.loc[i-1,'Daily Cases']
df.loc[i,'Daily Deaths 1'] = df.loc[i-1,'Daily Deaths']
df.loc[i,'Daily Cases 2'] = df.loc[i-2,'Daily Cases']
df.loc[i,'Daily Deaths 2'] = df.loc[i-2,'Daily Deaths']
df.loc[i,'Daily Cases 3'] = df.loc[i-3,'Daily Cases']
df.loc[i,'Daily Deaths 3'] = df.loc[i-3,'Daily Deaths']
df.loc[0,'Daily Cases 1'] = 0
df.loc[0,'Daily Deaths 1'] = 0
df.loc[0,'Daily Cases 2'] = 0
df.loc[0,'Daily Deaths 2'] = 0
df.loc[0,'Daily Cases 3'] = 0
df.loc[0,'Daily Deaths 3'] = 0
df.loc[1,'Daily Cases 1'] = df.loc[0,'Daily Cases']
df.loc[1,'Daily Deaths 1'] = df.loc[0,'Daily Deaths']
df.loc[1,'Daily Cases 2'] = 0
df.loc[1,'Daily Deaths 2'] = 0
df.loc[1,'Daily Cases 3'] = 0
df.loc[1,'Daily Deaths 3'] = 0
df.loc[2,'Daily Cases 1'] = df.loc[1,'Daily Cases']
df.loc[2,'Daily Deaths 1'] = df.loc[1,'Daily Deaths']
df.loc[2,'Daily Cases 2'] = df.loc[0,'Daily Cases']
df.loc[2,'Daily Deaths 2'] = df.loc[0,'Daily Deaths']
df.loc[2,'Daily Cases 3'] = 0
df.loc[2,'Daily Deaths 3'] = 0
return df
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
lagtime_mean_dict = lagtime_agg['mean'].astype('int32' ).to_dict(defaultdict(int))
del user_lecture_agg
del lagtime_agg
gc.collect()<categorify>
df_list = []
grouped = df_train.groupby(['Country_Region','Province_State'],as_index=False)
 for name,group in grouped:
df_tmp = df_train[(df_train["Country_Region"]==name[0])&(df_train["Province_State"] == name[1])].reset_index(drop = True)
 df_tmp = add_daily_measures(df_tmp)
 df_tmp = add_more_daily_measures(df_tmp)
 df_tmp['Lat'] = dict_geo[name][0]
df_tmp['Long'] = dict_geo[name][1]
df_tmp['population'] = dict_geo[name][2]
df_tmp['area'] = dict_geo[name][3]
df_tmp['popdensity'] = dict_geo[name][2]/dict_geo[name][3]
df_list.append(df_tmp)
 df_concat = pd.concat(df_list,ignore_index=True)
 df_train = df_concat
df_train,df_past,df_dev = train_dev_split(df_train,13)
 columns =['Daily Cases 3','Daily Cases 2','Daily Cases 1','Daily Deaths 3','Daily Deaths 2','Daily Deaths 1','Lat','Long','population','area','popdensity','lastconfirm','lastdeath','Days','Province_State', 'Country_Region','ConfirmedCases','Fatalities','Daily Cases','Daily Deaths']
df_train = df_train[columns]
df_dev = df_dev[columns]
attempt_no_agg=attempt_no_agg[attempt_no_agg['sum'] >1]
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()<categorify>
columns_test = ['Daily Cases 3','Daily Cases 2','Daily Cases 1','Daily Deaths 3','Daily Deaths 2','Daily Deaths 1','Lat','Long','population','area','popdensity','lastconfirm','lastdeath','Days']
def generate_test_feature(df):
alpha = []
beta = []
df_target = pd.DataFrame(columns = columns_test)
 df_adjust = df.tail(7)
 df_alpha = df_adjust['Daily Cases'].values
for i in range(6):
alpha.append(df_alpha[i+1]-df_alpha[i])
 df_beta = df_adjust['Daily Deaths'].values
for i in range(6):
beta.append(df_beta[i+1]-df_beta[i])
 df_tmp = df.tail(1)
 df_target['Daily Cases 3'] = df_tmp['Daily Cases 2']
df_target['Daily Cases 2'] = df_tmp['Daily Cases 1']
df_target['Daily Cases 1'] = df_tmp['Daily Cases']
df_target['Daily Deaths 3'] = df_tmp['Daily Deaths 2']
df_target['Daily Deaths 2'] = df_tmp['Daily Deaths 1']
df_target['Daily Deaths 1'] = df_tmp['Daily Deaths']
df_target['Lat'] = df_tmp['Lat']
df_target['Long'] = df_tmp['Long']
df_target['population'] = df_tmp['population']
df_target['area'] = df_tmp['area']
df_target['popdensity'] = df_tmp['popdensity']
df_target['lastconfirm'] = df_tmp['ConfirmedCases']
df_target['lastdeath'] = df_tmp['Fatalities']
df_target['Days'] = df_tmp['Days']+1
return df_target,sum(alpha),sum(beta )
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del user_prior_question_elapsed_time
gc.collect()<feature_engineering>
d = {'ForecastId':[], 'ConfirmedCases':[], 'Fatalities':[]}
submission_confirm = []
submission_death = []
china_train = df_train[(df_train['Country_Region']=='China')&(df_train['Days']>50)]
china_death = china_train.drop(['Country_Region','Province_State','ConfirmedCases','Fatalities','Daily Cases'], axis=1)
 china_death = china_death.values
china_confirm = china_train.drop(['Country_Region','Province_State','ConfirmedCases','Fatalities','Daily Deaths 3','Daily Deaths 2','Daily Deaths 1','Daily Deaths'], axis=1)
 china_confirm = china_confirm.values
df_train_noch = df_train[(df_train['Days']>50)&(df_train['Country_Region']!='China')]
train_death = df_train_noch.drop(['Country_Region','Province_State','ConfirmedCases','Fatalities','Daily Cases'], axis=1)
 print(train_death.columns)
 train_death = train_death.values
train_death = np.append(china_death,train_death,axis = 0)
 train_confirm = df_train_noch.drop(['Country_Region','Province_State','ConfirmedCases','Fatalities','Daily Deaths 3','Daily Deaths 2','Daily Deaths 1','Daily Deaths'], axis=1)
 print(train_confirm.columns)
 train_confirm = train_confirm.values
train_confirm = np.append(china_confirm,train_confirm,axis = 0)
 X_train_confirm, y_train_confirm = train_confirm[:,:-1], train_confirm[:,-1:]
X_train_death, y_train_death = train_death[:,:-1], train_death[:,-1:]
model1 = XGBRegressor(n_estimators = 1000,learning_rate = 0.1,tree_method='gpu_hist')
 model1.fit(X_train_confirm, y_train_confirm[:,0])
 model2 = XGBRegressor(n_estimators = 1000,learning_rate = 0.1,tree_method='gpu_hist')
 model2.fit(X_train_death, y_train_death[:,0])
 for country in df_train.Country_Region.unique() :
if country=='China':
alpha = 0.1
else:
alpha = 1
df_train1 = df_train[df_train["Country_Region"]==country]
for state in df_train1.Province_State.unique() :
df_train2 = df_train1[df_train1["Province_State"]==state]
df_train3 = df_train2.drop(['Country_Region','Province_State'], axis=1)
 df_overlay = df_dev[(df_dev["Country_Region"]==country)&(df_dev["Province_State"]==state)]
submission_confirm =submission_confirm + df_overlay['ConfirmedCases'].values.tolist()
submission_death =submission_death + df_overlay['Fatalities'].values.tolist()
for i in range(30):
df_target,confirm_fact,deaths_fact = generate_test_feature(df_train3)
 df_target_death = df_target[columns_test]
df_target_confirm = df_target[columns_test].drop(['Daily Deaths 3','Daily Deaths 2','Daily Deaths 1'], axis=1)
 target_confirm = df_target_confirm.values
target_death = df_target_death.values
if confirm_fact<0:
y_pred1 = abs(model1.predict(target_confirm)) *0.85*alpha
else:
y_pred1 = abs(model1.predict(target_confirm)) *alpha
if deaths_fact<0:
y_pred2 = abs(model2.predict(target_death)) *0.8*alpha
else:
y_pred2 = abs(model2.predict(target_death)) *alpha
submission_confirm.append(target_death[0][11]+int(y_pred1))
submission_death.append(target_death[0][12]+int(y_pred2))
target = target_death.flatten()
target = np.append(target,target[11]+int(y_pred1))
target = np.append(target,target[12]+int(y_pred2))
target = np.append(target,int(y_pred1))
target = np.append(target,int(y_pred2))
target = np.reshape(target,(1, 18))
last_index = df_train3.tail(1 ).index.values[0]
df_train3.loc[last_index+1] = target[0]
d['ConfirmedCases'] = submission_confirm
d['Fatalities'] = submission_death
a = np.arange(1,len(submission_death)+1,1)
 d['ForecastId'] = a
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]<choose_model_class>
print(model1.feature_importances_)
 print(model2.feature_importances_)
 df_submit = pd.DataFrame(d)
 print(df_submit.head())
 df_submit.to_csv(r'submission.csv', index=False )