Kaggle competition: COVID19 Global Forecasting (Week 4)
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]<feature_engineering>
import numpy as np
import pandas as pd
import datetime as dt
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
import pickle
<split>
warnings.filterwarnings('ignore' )
iter_test = env.iter_test()
prior_test_df = None<define_search_space>
train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv" )
N=[0.4,0.6]<data_type_conversions>
submission = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv" )
%%time
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 user_ids = prior_test_df['user_id'].values
targets = prior_test_df[target].values
for user_id, answered_correctly in zip(user_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
prior_test_df = test_df.copy()
question_len=len(test_df[test_df['content_type_id'] == 0])
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(question_len, dtype=np.int16)
 user_lecture_count = np.zeros(question_len, dtype=np.int16)
 user_sum = np.zeros(question_len, dtype=np.int16)
 user_count = np.zeros(question_len, dtype=np.int16)
 task_container_sum = np.zeros(question_len, dtype=np.int32)
 task_container_count = np.zeros(question_len, dtype=np.int32)
 task_container_std = np.zeros(question_len, dtype=np.float16)
 explanation_sum = np.zeros(question_len, dtype=np.int32)
 explanation_count = np.zeros(question_len, dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(question_len, dtype=np.int32)
 attempt_no_count = np.zeros(question_len, dtype=np.int16)
 lagtime = np.zeros(question_len, dtype=np.float32)
 lagtime2 = np.zeros(question_len, dtype=np.float32)
 lagtime3 = np.zeros(question_len, dtype=np.float32)
 i=0
for j,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(content_type_id==1):
x=1
if(content_type_id==0):
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
if(max_timestamp_u_dict2['max_time_stamp2'][user_id]==lagtime_mean2):
lagtime2[i]=lagtime_mean2
lagtime3[i]=lagtime_mean3
else:
lagtime2[i]=timestamp-max_timestamp_u_dict2['max_time_stamp2'][user_id]
if(max_timestamp_u_dict3['max_time_stamp3'][user_id]==lagtime_mean3):
lagtime3[i]=lagtime_mean3
else:
lagtime3[i]=timestamp-max_timestamp_u_dict3['max_time_stamp3'][user_id]
max_timestamp_u_dict3['max_time_stamp3'][user_id]=max_timestamp_u_dict2['max_time_stamp2'][user_id]
max_timestamp_u_dict2['max_time_stamp2'][user_id]=max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
else:
lagtime[i]=lagtime_mean
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime2[i]=lagtime_mean2
max_timestamp_u_dict2['max_time_stamp2'].update({user_id:lagtime_mean2})
 lagtime3[i]=lagtime_mean3
max_timestamp_u_dict3['max_time_stamp3'].update({user_id:lagtime_mean3})
 if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=delta_prior_question_elapsed_time_mean
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 i=i+1
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_sum'] = user_lecture_sum
test_df['user_interaction_count'] = user_lecture_count
test_df['user_interaction_timestamp_mean'] = test_df['timestamp']/user_lecture_count
test_df['user_correctness'] = user_sum / user_count
test_df['user_uncorrect_count'] =user_count-user_sum
test_df['user_correct_count'] =user_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_cor_count'] = task_container_sum
test_df['task_container_uncor_count'] =task_container_count-task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_true_count'] = explanation_sum
test_df['explanation_false_count'] = explanation_count-explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime2"]=lagtime2
test_df["lagtime3"]=lagtime3
test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('float16')
 test_df['lagtime']=test_df['lagtime']/(1000*3600)
 test_df.lagtime=test_df.lagtime.astype('float32')
 test_df['lagtime2']=test_df['lagtime2']/(1000*3600)
 test_df.lagtime2=test_df.lagtime2.astype('float32')
 test_df['lagtime3']=test_df['lagtime3']/(1000*3600)
 test_df.lagtime3=test_df.lagtime3.astype('float32')
 test_df['user_interaction_timestamp_mean']=test_df['user_interaction_timestamp_mean']/(1000*3600)
 test_df.user_interaction_timestamp_mean=test_df.user_interaction_timestamp_mean.astype('float32')
 test_df['user_correctness'].fillna(0.67, inplace=True)
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features])
 sub_preds += test_preds
test_df[target]=sub_preds / len(clfs)
 env.predict(test_df[['row_id', target]] )<install_modules>
train.columns = ['ForecastId', 'Province_State', 'Country_Region', 'Date', 'ConfirmedCases', 'Fatalities']
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1<set_options>
test = test[test['Date'] > train_date_end]
_ = np.seterr(divide='ignore', invalid='ignore' )<define_variables>
master = train.append(test )
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'<load_from_csv>
master['Province_State'] = master['Province_State'].fillna('')
 master['Country_State'] = master['Country_Region'].str.cat(master['Province_State'],sep=" " )
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()<load_from_csv>
lockdown_dates = pd.read_csv(".. /input/lockdown-dates-upd1/lockdown_dates.csv")
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )<categorify>
lockdown_dates['Country_State'] = lockdown_dates['Country_Region'].str.cat(lockdown_dates.Province_State.fillna(''),sep=" " )
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]<merge>
gov_stringency_data = pd.read_csv(".. /input/stringency-index/CSVDownload")
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )<groupby>
gov_stringency_data_use = gov_stringency_data[['CountryName','Date','StringencyIndex']]
gov_stringency_data_use['Date'] = pd.to_datetime(gov_stringency_data_use['Date'], format='%Y%m%d' ).dt.strftime('%Y-%m-%d' )
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()<data_type_conversions>
gov_stringency_data_use['CountryName'][gov_stringency_data_use['CountryName'] == 'United States'] = 'US'
gov_stringency_data_use['CountryName'][gov_stringency_data_use['CountryName'] == 'South Korea'] = 'Korea South'
gov_stringency_data_use['CountryName'][gov_stringency_data_use['CountryName'] == 'Myanmar'] = 'Burma'
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )<data_type_conversions>
gov_stringency_data_agg = gov_stringency_data_use[gov_stringency_data_use['StringencyIndex'] >= 50.0].groupby(['CountryName'])['StringencyIndex'].max().reset_index()
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )<filter>
gov_stringency_data_use = pd.merge(gov_stringency_data_use, gov_stringency_data_agg
,how='inner', on=['CountryName'])
 gov_stringency_data_use['StringencyIndex_diff_max'] = gov_stringency_data_use['StringencyIndex_x'] - gov_stringency_data_use['StringencyIndex_y']
train_lectures[train_lectures.user_id==5382]<filter>
gov_stringency_data_agg_use = gov_stringency_data_use[gov_stringency_data_use['StringencyIndex_diff_max'] >= -10.0].groupby(['CountryName'])['Date'].min().reset_index()
gov_stringency_data_agg_use.columns = ['Country_Region', 'Stringency_Date']
user_lecture_stats_part[user_lecture_stats_part.user_id==5382]<drop_column>
lockdown_dates_upd = pd.merge(lockdown_dates, gov_stringency_data_agg_use
,how='left', on=['Country_Region'] )
del(train_lectures )<data_type_conversions>
lockdown_dates_upd['Lockdown_date_use'] = ''
lockdown_dates_upd['Lockdown_date_use'][lockdown_dates_upd['Country_Region'].isin(['US','China','Australia'])== True] = lockdown_dates_upd['Lockdown_date']
lockdown_dates_upd['Lockdown_date_use'][lockdown_dates_upd['Country_Region'].isin(['US','China','Australia'])== False] = lockdown_dates_upd['Stringency_Date'].combine_first(lockdown_dates_upd['Lockdown_date'])
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 train_df['user_lecture_cumsum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_cumsum=train_df.user_lecture_cumsum.astype('int8')
 user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'] )<data_type_conversions>
def cc_model_poly(dataset,start_date,end_date,start_value,Country_State,knot_number):
dataset = dataset[(dataset['Date'] >= start_date)&(dataset['Date'] <= end_date)]
dataset['Date_diff'] =(((pd.to_datetime(dataset['Date'])- pd.to_datetime(start_date)).dt.days))
dataset['Date_diff_p2'] = dataset['Date_diff']**2
dataset['Date_diff_p3'] = dataset['Date_diff']**3
input_columns = ['Date_diff','Date_diff_p2','Date_diff_p3']
if len(dataset.dropna())> 5:
start_value = dataset['ConfirmedCases_inc'].values[0]
dataset['ConfirmedCases_inc_use'] = dataset['ConfirmedCases_inc_sm']
reg = LinearRegression().fit(dataset.dropna() [input_columns], dataset.dropna() ['ConfirmedCases_inc_use'])
 dataset['ConfirmedCases_pred'] = reg.predict(dataset[input_columns])
 filename = 'cc_'+Country_State+'_'+knot_number+'.sav'
pickle.dump(reg, open(filename, 'wb'))
else:
print('.. picking fallback cc model for knot '+knot_number+'.. ')
 reg = pickle.load(open('cc_China Hubei'+'_'+knot_number+'.sav', 'rb'))
dataset['ConfirmedCases_pred'] = reg.predict(dataset[input_columns])- reg.intercept_ + start_value
if knot_number == '2':
cap_upper = dataset['ConfirmedCases_inc'].mean() +(dataset['ConfirmedCases_inc'].std())
 cap_lower = dataset['ConfirmedCases_inc'].mean() -(dataset['ConfirmedCases_inc'].std())
 dataset['ConfirmedCases_pred'] = dataset['ConfirmedCases_pred'].clip(lower=cap_lower, upper=cap_upper)
 dataset['ConfirmedCases_pred'][dataset['ConfirmedCases_pred'] < 0] = 0
return(dataset )
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True)
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True )<groupby>
def cc_model_exp(dataset,start_date,end_date,start_value,Country_State,knot_number):
dataset = dataset[(dataset['Date'] >= start_date)&(dataset['Date'] <= end_date)]
dataset['Date_diff'] =(((pd.to_datetime(dataset['Date'])- pd.to_datetime(start_date)).dt.days))
if len(dataset.dropna())> 5:
dataset['ConfirmedCases_inc_use'] = dataset['ConfirmedCases_inc_sm'] + 0.0001
reg = LinearRegression().fit(dataset.dropna() [['Date_diff']], np.log(dataset.dropna() ['ConfirmedCases_inc_use']))
dataset['ConfirmedCases_pred'] = np.exp(reg.predict(dataset[['Date_diff']]))
filename = 'cc_'+Country_State+'_'+knot_number+'.sav'
pickle.dump(reg, open(filename, 'wb'))
else:
print('.. picking fallback cc model for knot '+knot_number+'.. ')
 reg = pickle.load(open('cc_China Hubei'+'_'+knot_number+'.sav', 'rb'))
dataset['ConfirmedCases_pred'] = np.exp(reg.predict(dataset[['Date_diff']])- reg.intercept_ + np.log(start_value))
if reg.coef_ > 0:
print('.. skipping exp model as the curve is not declining for knot '+knot_number+'.. ')
 dataset['ConfirmedCases_pred'] = dataset['ConfirmedCases_inc'][dataset['ConfirmedCases'] > 0].mean()
return(dataset )
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']<data_type_conversions>
def fa_model_lnr(dataset,Country_State):
dataset['ConfirmedCases_lagn'] = dataset['ConfirmedCases'].shift(periods=5)
 dataset['ConfirmedCases_pred_lagn'] = dataset['ConfirmedCases_pred'].shift(periods=5 ).fillna(0)
 if len(dataset.dropna())> 10:
reg = LinearRegression(fit_intercept=False ).fit(dataset.dropna() [['ConfirmedCases_lagn']], dataset.dropna() ['Fatalities'])
 dataset['Fatalities_pred'] = reg.predict(dataset[['ConfirmedCases_pred_lagn']])
 filename = 'fa_'+Country_State+'_'+'.sav'
pickle.dump(reg, open(filename, 'wb'))
else:
print('.. picking fallback fa model.. ')
 loaded_model = pickle.load(open('fa_China Hubei'+'_'+'.sav', 'rb'))
dataset['Fatalities_pred'] = loaded_model.predict(dataset[['ConfirmedCases_pred_lagn']])
 return(dataset )
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
train_df['lagtime'].fillna(0, inplace=True)
 train_df.lagtime=train_df.lagtime.astype('int32')
<categorify>
test_update = pd.DataFrame(columns = ['Province_State', 'Country_Region', 'Date', 'ConfirmedCases_pred' ,'Fatalities_pred'])
 for Country_State in ['China Hubei'] + master['Country_State'].unique().tolist() :
print('running for: '+Country_State)
 master_use = master[master['Country_State'] == Country_State]
master_use['ConfirmedCases_inc'] = master_use['ConfirmedCases'] - master_use['ConfirmedCases'].shift(periods=1 ).fillna(0)
 master_use['ConfirmedCases_inc_sm'] = master_use['ConfirmedCases_inc'].rolling(window=5,center=True,min_periods=1 ).mean()
st_dt = master_use['Date'][master_use['ConfirmedCases'] > 0].min()
mx_dt = master_use['Date'].max()
ld_dt = lockdown_dates_upd['Lockdown_date_use'][lockdown_dates_upd['Country_State'] == Country_State].values[0]
try:
if(ld_dt != '9999-12-31'):
add_wdw = 3
kt_1p =(pd.to_datetime(ld_dt)+dt.timedelta(days=14+add_wdw)).strftime("%Y-%m-%d")
 kt_1m =(pd.to_datetime(ld_dt)+dt.timedelta(days=14-add_wdw)).strftime("%Y-%m-%d")
 kt_2p =(pd.to_datetime(ld_dt)+dt.timedelta(days=28+add_wdw)).strftime("%Y-%m-%d")
 kt_2m =(pd.to_datetime(ld_dt)+dt.timedelta(days=28-add_wdw)).strftime("%Y-%m-%d")
 a = cc_model_poly(master_use, st_dt, kt_1p, 0, Country_State, '1')
 b = cc_model_poly(master_use, kt_1m, kt_2p, a.tail(1)['ConfirmedCases_pred'].values[0], Country_State, '2')
 c = cc_model_exp(master_use, kt_2m, mx_dt, b.tail(1)['ConfirmedCases_pred'].values[0], Country_State, '3')
 master_use = a.append(b ).append(c)
 master_use = master_use.groupby(['Province_State', 'Country_Region', 'Date', 'Country_State'])['ConfirmedCases', 'ConfirmedCases_pred', 'ConfirmedCases_inc', 'Fatalities'].mean().reset_index()
else:
master_use = cc_model_poly(master_use, st_dt, mx_dt, 0, Country_State, '1')
 master_use['ConfirmedCases_pred'] = master_use['ConfirmedCases_inc'].combine_first(master_use['ConfirmedCases_pred'])
 master_use['ConfirmedCases_pred'] = master_use['ConfirmedCases_pred'].cumsum()
master_use = fa_model_lnr(master_use, Country_State)
 master_use['Fatalities_pred'] = master_use['Fatalities'].combine_first(master_use['Fatalities_pred'])
 test_use = master_use[master_use['Date'] >= test_date_st]
test_update = test_update.append(test_use[['Province_State', 'Country_Region', 'Date', 'ConfirmedCases_pred', 'Fatalities_pred']])
 except:
print('exiting due to error')
 pass
lagtime_agg = train_df.groupby('user_id')['lagtime'].agg(['mean'])
 train_df['lagtime_mean'] = train_df['user_id'].map(lagtime_agg['mean'])
 train_df.lagtime_mean=train_df.lagtime_mean.astype('int32' )<groupby>
for Country_State in missing_inst:
print('running for: '+Country_State)
 master_use = master[master['Country_State'] == Country_State]
cc_ext = master_use['ConfirmedCases'].max()
fa_ext = master_use['Fatalities'].max()
master_use['ConfirmedCases_pred'] = cc_ext
master_use['Fatalities_pred'] = fa_ext
master_use['ConfirmedCases_pred'] = master_use['ConfirmedCases'].combine_first(master_use['ConfirmedCases_pred'])
 test_use = master_use[master_use['Date'] >= test_date_st]
test_update = test_update.append(test_use[['Province_State', 'Country_Region', 'Date', 'ConfirmedCases_pred', 'Fatalities_pred']] )
user_prior_question_elapsed_time = train_df[['user_id','prior_question_elapsed_time']].groupby(['user_id'] ).tail(1)
 user_prior_question_elapsed_time.columns = ['user_id', 'prior_question_elapsed_time']<feature_engineering>
test_update = test_update.groupby(['Province_State', 'Country_Region', 'Date'])['ConfirmedCases_pred','Fatalities_pred'].max().reset_index()
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']
train_df['delta_prior_question_elapsed_time'].fillna(0, inplace=True)
<data_type_conversions>
test_orig = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 test_orig['Province_State'] = test_orig['Province_State'].fillna('' )
train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )<data_type_conversions>
sub = pd.merge(test_orig[['ForecastId','Province_State','Country_Region', 'Date']], test_update[['Province_State','Country_Region', 'Date', 'ConfirmedCases_pred','Fatalities_pred']], how='left', on=['Province_State','Country_Region', 'Date'] )
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('int16')
<data_type_conversions>
sub['ConfirmedCases_pred'] = sub['ConfirmedCases_pred'].fillna(0)
 sub['Fatalities_pred'] = sub['Fatalities_pred'].fillna(0 )
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_cumsum'] = cum['cumsum']
train_df['user_correct_cumcount'] = cum['cumcount']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correct_cumsum'].fillna(0, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_cumcount=train_df.user_correct_cumcount.astype('int16')
 train_df.user_correct_cumsum=train_df.user_correct_cumsum.astype('int16' )<data_type_conversions>
sub = sub[['ForecastId','ConfirmedCases_pred','Fatalities_pred']]
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 train_df['lag'] = train_df.groupby('user_id')['prior_question_had_explanation'].shift()<data_type_conversions>
sub.to_csv("submission.csv",index=False )
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_cumsum'] = cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['explanation_mean'].fillna(0, inplace=True)
 train_df['explanation_cumsum'].fillna(0, inplace=True)
 train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_cumsum=train_df.explanation_cumsum.astype('int16' )<set_options>
for dirname, _, filenames in os.walk('.. '):
for filename in filenames:
if filename.lower().endswith('.sav'):
os.remove(os.path.join(dirname, filename))