Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<data_type_conversions>
import numpy as np
import pandas as pd
train_df['lagtime2'] = train_df.groupby('user_id')['timestamp'].shift(2)
 max_timestamp_u3 = train_df[['user_id','lagtime2']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u3.columns = ['user_id', 'max_time_stamp3']
max_timestamp_u3.user_id=max_timestamp_u3.user_id.astype('int32')
 train_df['lagtime2']=train_df['timestamp']-train_df['lagtime2']
lagtime_mean2=train_df['lagtime2'].mean()
train_df['lagtime2'].fillna(lagtime_mean2, inplace=True )<data_type_conversions>
X_train = pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv')
 X_test = pd.read_csv('.. /input/covid19-global-forecasting-week-4/test.csv')
 X_submission = pd.read_csv('.. /input/covid19-global-forecasting-week-4/submission.csv' )
train_df['lagtime2']=train_df['lagtime2']/(1000*3600)
 train_df.lagtime2=train_df.lagtime2.astype('float32' )<data_type_conversions>
X_train['Date'] = pd.to_datetime(X_train['Date'])
 X_test['Date'] = pd.to_datetime(X_test['Date'])
 X_test['Date']
train_df['lagtime3'] = train_df.groupby('user_id')['timestamp'].shift(3)
 train_df['lagtime3']=train_df['timestamp']-train_df['lagtime3']
lagtime_mean3=train_df['lagtime3'].mean()
train_df['lagtime3'].fillna(lagtime_mean3, inplace=True)
 train_df['lagtime3']=train_df['lagtime3']/(1000*3600)
 train_df.lagtime3=train_df.lagtime3.astype('float32' )<data_type_conversions>
print(X_train.Country_Region.nunique() )
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('float16' )<feature_engineering>
countries_no_province = [i for i in countries if i not in countries_with_provinces]
len(countries_no_province )
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']<data_type_conversions>
X_train['Province_State'] = X_train['Province_State'].fillna('unknown')
 X_test['Province_State'] = X_test['Province_State'].fillna('unknown' )
delta_prior_question_elapsed_time_mean=train_df['delta_prior_question_elapsed_time'].mean()
train_df['delta_prior_question_elapsed_time'].fillna(delta_prior_question_elapsed_time_mean, inplace=True)
 train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )<data_type_conversions>
X_train[X_train['Country_Region'].isin(countries_with_provinces)].groupby(['Country_Region'] ).agg({'Province_State':'nunique'} )
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 user_agg = train_df.groupby('user_id')['lag'].agg(['sum', 'count'] ).astype('int16')
 cum['cumsum'].fillna(0, inplace=True)
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_count'] = cum['cumsum']
train_df['user_uncorrect_count'] = cum['cumcount']-cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correctness'].fillna(0.67, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_count=train_df.user_correct_count.astype('int16')
 train_df.user_uncorrect_count=train_df.user_uncorrect_count.astype('int16' )<set_options>
X_train['Date'] = X_train['Date'].dt.strftime("%m%d")
 X_train['Date'] = X_train['Date'].astype(int)
 X_test['Date'] = X_test['Date'].dt.strftime("%m%d")
 X_test['Date'] = X_test['Date'].astype(int )
del cum
gc.collect()<data_type_conversions>
X_train['Province_State'] = X_train['Province_State'].fillna('unknown')
 X_test['Province_State'] = X_test['Province_State'].fillna('unknown' )
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16' )<data_type_conversions>
X_train['Province_State'] = X_train['Province_State'].astype('category')
 X_train['Country_Region'] = X_train['Country_Region'].astype('category')
 X_test['Province_State'] = X_test['Province_State'].astype('category')
 X_test['Country_Region'] = X_test['Country_Region'].astype('category')
 X_train
cum = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_true_count'] = cum['cumsum']
train_df['explanation_false_count'] = cum['cumcount']-cum['cumsum']
train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_true_count=train_df.explanation_true_count.astype('int16')
 train_df.explanation_false_count=train_df.explanation_false_count.astype('int16' )<set_options>
FEATURES = ['Date']
X_submission = pd.DataFrame(columns=['ForecastId', 'ConfirmedCases', 'Fatalities'])
 for i in tqdm(X_train.Country_Region.unique()):
z_train = X_train[X_train['Country_Region'] == i]
z_test = X_test[X_test['Country_Region'] == i]
for k in z_train.Province_State.unique() :
p_train = z_train[z_train['Province_State'] == k]
p_test = z_test[z_test['Province_State'] == k]
X_train_final = p_train[FEATURES]
y1 = p_train['ConfirmedCases']
y2 = p_train['Fatalities']
model = xgb.XGBRegressor(n_estimators=2000)
 model.fit(X_train_final, y1)
 ConfirmedCasesPreds = model.predict(p_test[FEATURES])
 model.fit(X_train_final, y2)
 FatalitiesPreds = model.predict(p_test[FEATURES])
 p_test['ConfirmedCases'] = ConfirmedCasesPreds
p_test['Fatalities'] = FatalitiesPreds
X_submission = pd.concat([X_submission, p_test[['ForecastId', 'ConfirmedCases', 'Fatalities']]], axis=0)
del cum
gc.collect()<categorify>
X_submission.to_csv('submission.csv', index=False )