Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<load_from_csv>
warnings.filterwarnings('ignore')
 daybasecount = 4
baseday = 98 - float(daybasecount-1)/2.
 exponent = 1./float(daybasecount)
 fatalityBaseDayShift = 10
maxincrease = 140
maxDeadPrDay = 1500
data_cudf = cu.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 train = data_cudf.to_pandas()
del data_cudf
train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]<prepare_x_and_y>
dftrain = pd.read_csv('.. /input/covid19-global-forecasting-week-3/train.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'])
 dftest = pd.read_csv('.. /input/covid19-global-forecasting-week-3/test.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'])
 ppp_tabel = pd.read_csv('.. /input/country-ppp/Country_PPP.csv', sep='\s+')
 ppp_tabel.drop('Id', 1,inplace=True)
 ppp_tabel = ppp_tabel.append({'Country' : 'Burma' , 'ppp' : 8000} , ignore_index=True)
 ppp_tabel = ppp_tabel.append({'Country' : 'MS_Zaandam' , 'ppp' : 40000} , ignore_index=True)
 ppp_tabel = ppp_tabel.append({'Country' : 'West_Bank_and_Gaza' , 'ppp' : 20000} , ignore_index=True)
 ppp_tabel["Country"].replace('_',' ', regex=True,inplace=True)
 ppp_tabel["Country"].replace('United States','US', regex=True,inplace=True)
 ppp_tabel.rename(columns={'Country':'Country_Region'},inplace=True)
 ppp_tabel.sort_values('Country_Region',inplace=True )
features = [c for c in train.columns if 'feature' in c]
for i in features:
x = train[i].mean()
train[i] = train[i].fillna(x)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T<choose_model_class>
dftrain['Dayofyear'] = dftrain['Date'].dt.dayofyear
dftest['Dayofyear'] = dftest['Date'].dt.dayofyear
dftest['Expo'] = dftest['Dayofyear']-baseday
print(dftrain.tail(5))
dftest = dftest.merge(dftrain[['Country_Region','Province_State','Date','ConfirmedCases','Fatalities']]
, on=['Country_Region','Province_State','Date'], how='left', indicator=True)
SEED = 1111
np.random.seed(SEED)
 def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
epochs = 200
batch_size = 4096
hidden_units = [160, 160, 160]
dropout_rates = [0.2, 0.2, 0.2, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-2
tf.keras.backend.clear_session()
tf.random.set_seed(SEED)
 clf = create_mlp(len(features), 5, hidden_units, dropout_rates, label_smoothing, learning_rate)
 clf.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, verbose=2)
 clf.save(f'model_v12.h5' )<define_variables>
grouped=dftrain.groupby(['Country_Region','Province_State'] ).tail(daybasecount*2)
 grouped=grouped.groupby(['Country_Region','Province_State'] ).head(daybasecount)
 grouped.drop(['FatalityBasis'],axis=1,inplace=True)
 to_sum = ['NewCases','NewFatalities']
grouped1 = grouped.groupby(['Country_Region'])[to_sum].sum()
grouped1.rename(columns={'NewCases':'NewCases1','NewFatalities':'NewFatalities1'}, inplace=True)
 grouped = pd.merge(grouped1, grouped2, on=['Country_Region'])
 grouped['CasesIncreasePct'] = 100*(grouped['NewCases2']/grouped['NewCases1']-1)
 mask = grouped['CasesIncreasePct'] > maxincrease
grouped.loc[mask,'CasesIncreasePct'] = maxincrease
mask = grouped['CasesIncreasePct'] < 0
grouped.loc[mask,'CasesIncreasePct'] = 0
mask = grouped['CasesIncreasePct'].isnull()
grouped.loc[mask,'CasesIncreasePct'] = 0
grouped['Factor'] =(grouped['CasesIncreasePct']/100+1)**exponent
grouped = pd.merge(grouped, ppp_tabel, on=['Country_Region'])
 grouped['ppp'] = grouped['ppp']/10000.
 if False:
mask =(grouped['FatalityPct2'] > 9)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 5
mask =(grouped['FatalityPct2'] < 5)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 5
mask =(grouped['FatalityPct2'] > 6)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 6
mask =(grouped['FatalityPct2'] < 1.5)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 1.5
mask =(grouped['FatalityPct2'] >(9.5 - 0.43*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(9.5 - 0.43*grouped['ppp'])
 mask =(grouped['FatalityPct2'] <(5.6 - 0.5*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(5.6 - 0.5*grouped['ppp'])
 mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 7
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 4
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(7.5 - 0.5*grouped['ppp'])
 else:
mask =(grouped['FatalityPct2'] > 4)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 3
mask =(grouped['FatalityPct2'] < 1)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 2
mask =(grouped['FatalityPct2'] > 1.5)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 1.5
mask =(grouped['FatalityPct2'] < 0.5)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 0.5
mask =(grouped['FatalityPct2'] >(4.5 - 0.43*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(4.5 - 0.43*grouped['ppp'])
 mask =(grouped['FatalityPct2'] <(1.1 - 0.1*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(1.1 - 0.1*grouped['ppp'])
 mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 3
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 1
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(2.6 - 0.23*grouped['ppp'] )
th = 0.503
f_mean = np.mean(train[features[1:]].values,axis=0)
 env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.median(clf(x_tt))
pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<load_from_csv>
dftest.drop('_merge',axis=1,inplace= True)
 dftest = dftest.merge(grouped[['Country_Region','FatalityPct2','Factor']], on=['Country_Region'], how='left')
 dftest = dftest.merge(grouped_gem[['Province_State','Country_Region','ConfirmedCases_base','ConfirmedCases_init','NewCases_base','Fatalities_init','FatalityBasis']], on=['Province_State','Country_Region'], how='left')
SEED = 1111
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 train = pd.read_feather('.. /input/janestreet-save-as-feather/train.feather')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T<define_variables>
dftest['ConfirmedCases_shift'] = dftest.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases']].transform(lambda x: x.shift(1))
mask = dftest['ConfirmedCases'].isnull()
dftest.loc[mask,'NewCases'] = dftest.loc[mask,'NewCases_base']*(dftest.loc[mask,'Factor']**dftest.loc[mask,'Expo'])
 dftest['NewCases_cum'] = dftest.groupby(['Country_Region', 'Province_State'])[['NewCases']].cumsum()
dftest.loc[mask,'ConfirmedCases'] = dftest.loc[mask,'ConfirmedCases_init'] + dftest.loc[mask,'NewCases_cum']
mask3 = dftest['ConfirmedCases'] > 400000
dftest.loc[mask3,'FatalityPct2'] = dftest.loc[mask3,'FatalityPct2']*0.7
mask4 = dftest['ConfirmedCases'] > 800000
dftest.loc[mask4,'FatalityPct2'] = dftest.loc[mask4,'FatalityPct2']*0.7
dftest['FatalityBasis'] = dftest.groupby(['Country_Region', 'Province_State'])[
['ConfirmedCases']].transform(lambda x: x.shift(10))
dftest.loc[mask,'NewFatalities'] = dftest.loc[mask,'FatalityBasis'] * dftest.loc[mask,'FatalityPct2']/100
mask2 = dftest['NewFatalities'] > maxDeadPrDay
dftest.loc[mask2,'NewFatalities'] = maxDeadPrDay
dftest['NewFatalities_cum'] = dftest.groupby(['Country_Region', 'Province_State'])[['NewFatalities']].cumsum()
dftest.loc[mask,'Fatalities'] = dftest.loc[mask,'Fatalities_init'] + dftest.loc[mask,'NewFatalities_cum']
features1 = [f'feature_{i}' for i in range(71)]
features2 = [f'feature_{i}' for i in range(71, 130)]<prepare_x_and_y>
dftest.drop(['Dayofyear',
'Expo','FatalityPct2', 'Factor',
'ConfirmedCases_base', 'ConfirmedCases_init',
'NewCases_base', 'Fatalities_init', 'FatalityBasis',
'ConfirmedCases_shift',
'NewCases', 'NewCases_cum', 'NewFatalities','NewFatalities_cum'],axis=1,inplace=True)
 final = dftest.groupby(['Country_Region','Province_State'] ).tail(1)
 dftest.drop(['Province_State'],axis=1,inplace=True)
 dftest.rename(columns={'Province_State_orig':'Province_State'},inplace=True )
X_train = [train.loc[:, features1].values,
train.loc[:, features2].values]<define_variables>
mask = dftest["ConfirmedCases"].isnull()
print(mask.sum())
 errors = dftest.loc[mask]
print(errors)
 mask = dftest["Fatalities"].isnull()
print(mask.sum())
 errors = dftest.loc[mask]
print(errors)
 dftest.drop(['Province_State','Country_Region','Date'],axis=1,inplace=True)
 print("dftest columns =",dftest.columns)
assert len(features)== len(features1)+ len(features2 )<normalization>
dftest.ForecastId = dftest.ForecastId.astype('int')
 dftest['ConfirmedCases'] = dftest['ConfirmedCases'].round().astype(int)
 dftest['Fatalities'] = dftest['Fatalities'].round().astype(int)
 dftest.to_csv('submission.csv', index=False)