Kaggle competition: COVID19 Global Forecasting (Week 4)
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()<data_type_conversions>
warnings.filterwarnings('ignore')
 sub0 = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv')
 train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv', parse_dates=['Date'])
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv', parse_dates=['Date'])
 train.shape, test.shape
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )<data_type_conversions>
know = test[test.Date <= train.Date.max() ]
not_know = test[test.Date > train.Date.max() ]
know = know.merge(train, on=['Date','Country_Region','Province_State'], how='left')
 know.head()
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )<drop_column>
train['days'] =(train['Date'] - train.Date.min() ).dt.days
train['location'] = train['Country_Region'] + ' ' + train['Province_State'].fillna('')
 train['location'] = train['location'].str.strip()
not_know['days'] =(not_know['Date'] - train.Date.min() ).dt.days
not_know['location'] = not_know['Country_Region'] + ' ' + not_know['Province_State'].fillna('')
 not_know['location']= not_know['location'].str.strip()
del(train_lectures)
 gc.collect()<categorify>
def get_param(loc):
_ = train[train.location == loc]
_['diff'] = _.ConfirmedCases.diff()
_['pct'] = _.ConfirmedCases.pct_change()
initial_speed = _.loc[_.ConfirmedCases.diff().argmax() ,'pct']
initial_mid = _.loc[_.ConfirmedCases.diff().argmax() , 'days']
initial_max = _.ConfirmedCases.max() * 2.1
return initial_speed, initial_mid, initial_max
get_param('Finland' )
user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'])
 user_lecture_agg=user_lecture_agg.astype('int16' )<data_type_conversions>
loc_list = train.location.unique()
all_param = pd.DataFrame(index=loc_list, columns=['k','x_0','y_max'])
 for loc in loc_list:
_ = train[train.location == loc]
nn = not_know[not_know.location == loc]
initial_max = _.ConfirmedCases.max() *2
x = _.days
y1 = _.ConfirmedCases
try:
popt, pcov = opt.curve_fit(log_curve, x, y1, p0 = get_param(loc))
popt[2] = max(popt[2],get_param(loc)[2]/2)
 except RuntimeError:
popt = get_param(loc)
 y1_hat = log_curve(nn.days, *popt)
 not_know.loc[y1_hat.index, 'ConfirmedCases'] = y1_hat
all_param.loc[loc, 'k'] = popt[0]
all_param.loc[loc, 'x_0'] = popt[1]
all_param.loc[loc, 'y_max'] = popt[2]
print('Done' )
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['user_interaction_count'] = cum['cumcount']
train_df['user_interaction_timestamp_mean'] = train_df['timestamp']/cum['cumcount']
train_df['user_lecture_sum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_sum=train_df.user_lecture_sum.astype('int16')
 train_df.user_interaction_count=train_df.user_interaction_count.astype('int16')
 train_df['user_interaction_timestamp_mean']=train_df['user_interaction_timestamp_mean']/(1000*3600)
 train_df.user_interaction_timestamp_mean=train_df.user_interaction_timestamp_mean.astype('float32')
<set_options>
all_param.loc['Finland', 'y_max']
del cum
gc.collect()<train_model>
latest = train[train.Date == train.Date.max() ]
latest['DeathRate'] = latest['Fatalities'] / latest['ConfirmedCases']
not_know2 = not_know.merge(latest[['location','DeathRate']], on='location')
 not_know2['Fatalities'] = not_know2['ConfirmedCases'] * not_know2['DeathRate']*1.1
print('start handle train_df...' )<data_type_conversions>
not_know2['ConfirmedCases'] = not_know2['ConfirmedCases'].round()
not_know2['Fatalities'] = not_know2['Fatalities'].round()
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True )<groupby>
sub1 = pd.concat([know[['ForecastId', 'ConfirmedCases','Fatalities']],
not_know2[['ForecastId', 'ConfirmedCases','Fatalities']]])
 sub1=sub1.sort_values('ForecastId' ).reset_index(drop=True)
 sub1.to_csv('submission.csv', index=False )