Kaggle competition: COVID19 Global Forecasting (Week 4)
questions_df['content_sub_bundle'] = questions_df['bundle_id'].map(bundle_agg['count'] ).astype('int8' )<set_options>
PATH ='/kaggle/input/covid19-global-forecasting-week-4'
datatrain = pd.read_csv(f'{PATH}/train.csv')
 datatest = pd.read_csv(f'{PATH}/test.csv' )
questions_df['tags'].fillna('188', inplace=True )<string_transform>
date = pd.to_datetime(datatrain["Date"])
 datet = pd.to_datetime(datatest["Date"])
 print(date)
def gettags(tags,num):
tags_splits=tags.split(" ")
 result=''
for t in tags_splits:
x=int(t)
 if(x<32*(num+1)and x>=32*num):
result=result+' '+t
return result<categorify>
ldate = int(len(date))
ldatet = int(len(datet))
print("Length of training- date is", ldate)
 print("Length of test- date is", ldatet )
for num in range(0,6):
questions_df["tags"+str(num)] = questions_df["tags"].apply(lambda row: gettags(row,num))
le = LabelEncoder()
le.fit(np.unique(questions_df['tags'+str(num)].values))
questions_df['tags'+str(num)]=questions_df[['tags'+str(num)]].apply(le.transform )<data_type_conversions>
m = []
d = []
for i in range(0,ldate):
dx =(date[i].strftime("%d"))
mx =(date[i].strftime("%m"))
m.append(int(mx))
d.append(int(dx))
mt = []
dt = []
for i in range(0,ldatet):
dtx =(datet[i].strftime("%d"))
mtx =(datet[i].strftime("%m"))
mt.append(int(mtx))
dt.append(int(dtx))
questions_df_dict = {
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
}
questions_df = questions_df.astype(questions_df_dict )<drop_column>
train = datatrain
test = datatest
questions_df.drop(columns=['tags'], inplace=True )<data_type_conversions>
train.insert(6,"Month",m,False)
 train.insert(7,"Day",d,False)
 test.insert(4,"Month",mt,False)
 test.insert(5,"Day",dt,False )
questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
<load_from_csv>
print("Datatrain")
 traindays = datatrain['Date'].nunique()
print("Number of Country_Region: ", datatrain['Country_Region'].nunique())
 print("Number of Province_State: ", datatrain['Province_State'].nunique())
 print("Number of Days: ", traindays)
 notrain = datatrain['Id'].nunique()
print("Number of datapoints in train:", notrain)
 lotrain = int(notrain/traindays)
 print("L Trains:", lotrain)
questions_cmnts = pd.read_csv(
 '.. /input/2020-r3id-clustering-question-tags/question_cmnts.csv',
usecols=[1,2],
dtype={'question_id': 'int16','community': 'int8'}
)<merge>
print("Datatest")
 testdays = datatest['Date'].nunique()
print("Number of Days: ", testdays)
 notest = datatest['ForecastId'].nunique()
print("Number of datapoints in test:", notest)
 lotest = int(notest/testdays)
 print("L Test:", lotest)
questions_df = pd.merge(questions_df, questions_cmnts, on='question_id', how='left',right_index=True )<rename_columns>
zt = datet[0]
daycount = []
for i in range(0,lotrain):
for j in range(1,traindays+1):
daycount.append(j)
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )<merge>
for i in range(traindays):
if(zt == date[i]):
zx = i
print(zx)
 daytest = []
for i in range(0,lotest):
for j in range(1,testdays+1):
jr = zx + j
daytest.append(jr)
questions_df = pd.merge(questions_df, content_explation_agg, on='content_id', how='left',right_index=True)
<drop_column>
train.insert(8,"DayCount",daycount,False)
 test.insert(6,"DayCount",daytest,False )
del content_explation_agg<data_type_conversions>
traincount = int(len(train["Date"]))
testcount = int(len(test["Date"]))
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16')
 questions_df['content_uncorrect_count'] = questions_df['content_id'].map(content_agg['count']-content_agg['sum'] ).astype('int32')
 questions_df['content_correct_count'] = questions_df['content_id'].map(content_agg['sum'] ).astype('int32' )<data_type_conversions>
train.Province_State = train.Province_State.fillna(0)
 empty = 0
for i in range(0,traincount):
if(train.Province_State[i] == empty):
train.Province_State[i] = train.Country_Region[i]
questions_df['content_elapsed_time_mean'] = questions_df['content_id'].map(content_elapsed_time_agg['mean'])
 questions_df.content_elapsed_time_mean=questions_df.content_elapsed_time_mean.astype('float16')
 questions_df['content_had_explanation_mean'] = questions_df['content_id'].map(content_had_explanation_agg['mean'])
 questions_df.content_had_explanation_mean=questions_df.content_had_explanation_mean.astype('float16' )<drop_column>
test.Province_State = test.Province_State.fillna(0)
 empty = 0
for i in range(0,testcount):
if(test.Province_State[i] == empty):
test.Province_State[i] = test.Country_Region[i]
del content_elapsed_time_agg
del content_had_explanation_agg
gc.collect()<categorify>
label = preprocessing.LabelEncoder()
train.Country_Region = label.fit_transform(train.Country_Region)
 train.Province_State = label.fit_transform(train.Province_State )
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )<data_type_conversions>
test.Country_Region = label.fit_transform(test.Country_Region)
 test.Province_State = label.fit_transform(test.Province_State)
part_agg = questions_df.groupby('part')['content_uncorrect_count'].agg(['sum'])
 questions_df['part_uncor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 part_agg = questions_df.groupby('part')['content_correct_count'].agg(['sum'])
 questions_df['part_cor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32' )<categorify>
X = np.c_[train["Province_State"], train["Country_Region"], train["DayCount"], train["Month"], train["Day"]]
Xt = np.c_[test["Province_State"], test["Country_Region"], test["DayCount"], test["Month"], test["Day"]]
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness_mean'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness_mean=questions_df.bundle_correctness_mean.astype('float16')
<data_type_conversions>
Y1 = train["ConfirmedCases"]
Y2 = train["Fatalities"]
<drop_column>
nest = [23,24,25]
alpha = [0]
gam = [0]
lr = [2,3,4]
dep = [23]
mscore = 1
for a in range(0,len(alpha)) :
for b in range(0,len(gam)) :
for c in range(0,len(lr)) :
for d in range(0,len(dep)) :
for e in range(0,len(nest)) :
a0 = alpha[a]; g0 = gam[b]; l0 = lr[c]/100;
d0 = dep[d]
n0 = nest[e] * 100
xgb = XGBRegressor(n_estimators = n0 , alpha = a0, gamma = g0, learning_rate = l0, random_state = 42 , max_depth = d0)
 xgb.fit(X,Y1)
 yscore = xgb.predict(X)
 yscore = np.round(yscore)
 ascore = mean_squared_error(yscore,Y1)
 print("Accuracy", ascore)
 if(ascore<mscore):
mscore = ascore
best = [a0,g0,l0,d0,n0]
print("New Max Accuracy is ",mscore, "alpha", a0, "for gamma", g0, "learning rate", l0, "depth", d0, "n_estimators", n0);
print("Best Estimators are, alpha:", best[0], "Gamma:", best[1], "learning rate", best[2], "depth", best[3], "n_estimators", best[4]);
del content_agg
del bundle_agg
del part_agg
gc.collect()<define_variables>
regr = XGBRegressor(n_estimators = best[4] , alpha = best[0], gamma = best[1], learning_rate = best[2], random_state = 42 , max_depth = best[3])
 regr1 = XGBRegressor(n_estimators = best[4] , alpha = best[0], gamma = best[1], learning_rate = best[2], random_state = 42 , max_depth = best[3] )
features_dict = {
'timestamp':'float16',
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'task_container_correctness':'float16',
'task_container_std':'float16',
'task_container_cor_count':'int32',
'task_container_uncor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'part_correctness_std':'float16',
'part_uncor_count':'int32',
'part_cor_count':'int32',
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
'part_bundle_id':'int32',
'content_sub_bundle':'int8',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16',
}
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags0',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'part_bundle_id',
'content_sub_bundle',
'prior_question_had_explanation',
]
features=list(features_dict.keys())
<init_hyperparams>
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 300,
'max_bin':450,
'feature_fraction': 0.52,
'bagging_fraction': 0.52,
'objective': 'binary',
'learning_rate': 0.05,
"boosting_type": "gbdt",
"metric": 'auc',
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df[1200*10000:2*1200*10000]
print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.08)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 print('pd.merge(train_df_clf, questions_df)')
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 print('valid_df')
 valid_df=train_df_clf.sample(frac=0.1)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('train_df_clf length：',len(train_df_clf))
print('valid_df length：',len(valid_df)) 
<drop_column>
regr.fit(X,Y1.ravel())
 yscore = regr.predict(X)
 ascore = mean_squared_error(yscore,Y1)
 print("Training(ConfirmedCases)- Mean Squared Error is: ",ascore )
del train_df_clf
del valid_df
gc.collect()<prepare_x_and_y>
ypred = regr.predict(Xt)
 ypred = pd.DataFrame({'ConfirmedCases' : ypred})
 ypred = round(ypred)
 ypred.head(20 )
for i in range(0,num):
X_train_np = trains[i][features].values.astype(np.float32)
 X_valid_np = valids[i][features].values.astype(np.float32)
 tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))
va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))
del trains
del valids
del X_train_np
del X_valid_np
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=5000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 fig,ax = plt.subplots(figsize=(15,15))
lgb.plot_importance(model, ax=ax,importance_type='gain',max_num_features=50)
 plt.show()
del tr_data
del va_data
gc.collect() 
<data_type_conversions>
regr1.fit(X,Y2.ravel())
 ypred2= regr1.predict(Xt)
 yptest = regr1.predict(X)
 yptest = np.round(yptest)
 ascore = mean_squared_error(yptest,Y2)
 print("Training -(Fatalities)Mean Squared Error is", ascore )
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int)) 
<data_type_conversions>
ypred2 = pd.DataFrame({'Fatalities' : ypred2})
 ypred2 = round(ypred2)
 ypred2.head(20 )
del user_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()<data_type_conversions>
ypc = pd.DataFrame()
forecast = test["ForecastId"]
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del user_lecture_agg
gc.collect()<categorify>
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
max_timestamp_u_dict2=max_timestamp_u2.set_index('user_id' ).to_dict()
max_timestamp_u_dict3=max_timestamp_u3.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del max_timestamp_u2
del max_timestamp_u3
del user_prior_question_elapsed_time
gc.collect()<categorify>
ypc.insert(0,"ForecastId",forecast,False)
 ypc.insert(1,"ConfirmedCases",ypred,False)
 ypc.insert(2, "Fatalities",ypred2,False )
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()<feature_engineering>
ypc.to_csv('submission.csv', index=False )