Kaggle competition: Jane Street Market Prediction
train.isnull().sum()<count_missing_values>
train = pd.read_csv(".. /input/jane-street-market-prediction/train.csv" )
train.isnull().sum()<merge>
train = train.query('date > 85' ).reset_index(drop = True )
cond1 = train.Fatalities >=1
cond2 = train.ConfirmedCases >= 1
train_Fatal = train[['Country_Region','Date']][(cond1)]
train_Confirm = train[['Country_Region','Date']][(cond2)]
Confirm_Min_Max = train_Confirm.groupby("Country_Region", as_index=False)["Date"].agg(["min","max"])
 Fatal_Min_Max = train_Fatal.groupby("Country_Region", as_index=False)["Date"].agg(["min","max"])
 Country_Dates = pd.merge(Confirm_Min_Max, Fatal_Min_Max, how='left', on=['Country_Region'])
 Country_Dates = Country_Dates.rename(columns={'min_x': 'fst_Confirmed', 'max_x': 'last_Confirmed', 'min_y': 'fst_Fatal','max_y': 'last_Fatal'})
 Country_Dates['Fatality_Days'] = pd.to_datetime(Country_Dates['last_Fatal'])- pd.to_datetime(Country_Dates['fst_Fatal'])
 Country_Dates = Country_Dates.reset_index() 
<feature_engineering>
features_mean = train.loc[:, features].mean()
Country_Dates['Fatality_Days'] = Country_Dates['Fatality_Days'] / np.timedelta64(1, 'D')
 Country_Dates.head(5 )<data_type_conversions>
train.fillna(train.mean() , inplace=True )
cond1 = train.Date == '2020-04-12'
Country_Counts = train[['Country_Region','Date','Fatalities','ConfirmedCases']][(cond1)]
df_Country2 = pd.merge(Country_Dates, Country_Counts, how='left', on=['Country_Region'])
 df_Country2['Fatality_Rate'] = round(df_Country2['Fatalities']/pd.to_numeric(df_Country2['Fatality_Days']),2)
 df_Country2['Fatality_Rate'] = round(df_Country2['Fatalities']/pd.to_numeric(df_Country2['Fatality_Days']),2)
 df_Country2['Fatality_Rate'] = df_Country2['Fatality_Rate'].fillna(0)
 df_Country2 = df_Country2.replace([np.inf, -np.inf], np.nan)
 df_Country3 = df_Country2[df_Country2['Fatality_Rate'] > 0].groupby(['Country_Region'])[["Fatality_Rate"]].sum().nlargest(8,'Fatality_Rate')
 df_Country3 = df_Country3.reset_index()<groupby>
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']
X_train = train.loc[:, features].values
y_train = np.stack([train[c] for c in resp_cols] ).T
print(X_train.shape, y_train.shape)
 del train
confirmed_China = train[train['Country_Region']=='China'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_China = train[train['Country_Region']=='China'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_China = confirmed_China.join(fatalities_China)
 confirmed_Italy = train[train['Country_Region']=='Italy'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Italy = train[train['Country_Region']=='Italy'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Italy = confirmed_Italy.join(fatalities_Italy)
 confirmed_Spain = train[train['Country_Region']=='Spain'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Spain = train[train['Country_Region']=='Spain'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Spain = confirmed_Spain.join(fatalities_Spain)
 confirmed_Iran = train[train['Country_Region']=='Iran'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Iran = train[train['Country_Region']=='Iran'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Iran = confirmed_Iran.join(fatalities_Iran)
 confirmed_France = train[train['Country_Region']=='France'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_France = train[train['Country_Region']=='France'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_France = confirmed_France.join(fatalities_France)
 confirmed_Netherlands = train[train['Country_Region']=='Netherlands'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_Netherlands = train[train['Country_Region']=='Netherlands'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_Netherlands = confirmed_Netherlands.join(fatalities_Netherlands)
 confirmed_UK = train[train['Country_Region']=='United Kingdom'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_UK = train[train['Country_Region']=='United Kingdom'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_UK = confirmed_UK.join(fatalities_UK)
 confirmed_USA = train[train['Country_Region']=='US'].groupby(['Date'] ).agg({'ConfirmedCases':['sum']})
 fatalities_USA = train[train['Country_Region']=='US'].groupby(['Date'] ).agg({'Fatalities':['sum']})
 total_USA = confirmed_USA.join(fatalities_USA)
<merge>
df_pop_sel = df_pop[['Country(or dependency)','Population(2020)','Density(P/Km²)','Land Area(Km²)','Med.Age','Urban Pop %']]
df_pop_sel.columns = ['Country_Region', 'Population(2020)', 'Density(P/Km²)','Land Area(Km²)', 'Med.Age', 'Urban Pop %']
df_pop_sel['Urban Pop %'] = df_pop_sel['Urban Pop %'].str.rstrip('%')
 df_pop_sel.loc[df_pop_sel['Country_Region']=='United States', 'Country_Region'] = 'US'
df_country_pop = pd.merge(df_pop_sel, df_Country, how='inner', on=['Country_Region'])
 df_country_pop<feature_engineering>
HIDDEN_LAYER_1 = [256, 256]
HIDDEN_LAYER_2 = [160, 160, 160]
HIDDEN_LAYER_3 = [128, 128, 128, 128]
TARGET_NUM = 5
input = tf.keras.layers.Input(shape=(X_train.shape[1],))
x1 = tf.keras.layers.BatchNormalization()(input)
 x1 = tf.keras.layers.Dropout(0.25 )(x1)
 for units in HIDDEN_LAYER_1:
x1 = tf.keras.layers.Dense(units )(x1)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.Activation(tf.keras.activations.swish )(x1)
 x1 = tf.keras.layers.Dropout(0.25 )(x1)
 x2 = tf.keras.layers.BatchNormalization()(input)
 x2 = tf.keras.layers.Dropout(0.25 )(x2)
 for units in HIDDEN_LAYER_2:
x2 = tf.keras.layers.Dense(units )(x2)
 x2 = tf.keras.layers.BatchNormalization()(x2)
 x2 = tf.keras.layers.Activation(tf.keras.activations.swish )(x2)
 x2 = tf.keras.layers.Dropout(0.25 )(x2)
 x3 = tf.keras.layers.BatchNormalization()(input)
 x3 = tf.keras.layers.Dropout(0.25 )(x3)
 for units in HIDDEN_LAYER_3:
x3 = tf.keras.layers.Dense(units )(x3)
 x3 = tf.keras.layers.BatchNormalization()(x3)
 x3 = tf.keras.layers.Activation(tf.keras.activations.swish )(x3)
 x3 = tf.keras.layers.Dropout(0.25 )(x3)
 x = tf.keras.layers.concatenate([x1, x2, x3])
 output = tf.keras.layers.Dense(TARGET_NUM )(x)
 model = tf.keras.models.Model(inputs=input, outputs=output)
 model.compile(
 optimizer = tf.optimizers.Adam(learning_rate=1e-3),
metrics = tf.keras.metrics.MeanSquaredError(name='mean_squared_error'),
loss = "MSE", 
)
 display(plot_model(model))
print('Done!' )
train_fil['Date'] = pd.to_datetime(train_fil['Date'])
 train_fil['Day_num'] = preprocessing.LabelEncoder().fit_transform(train_fil.Date)
 train_fil['Day'] = train_fil['Date'].dt.day
train_fil['Month'] = train_fil['Date'].dt.month
train_fil['Year'] = train_fil['Date'].dt.year
train_fil_copy = train_fil
train_fil['Province_State'].fillna("None", inplace=True)
 train_fil[train_fil.iloc[:,0:].isnull().any(axis = 1)].iloc[:,0:].head()<merge>
print('Train NN...')
 history = model.fit(
 x = X_train,
y = y_train,
epochs=200,
batch_size=4096, 
)
 models = []
models.append(model)
 print('Done!' )
df_train_pop = train_fil.merge(covid_pop_sel, left_on = ['Country_Region','Province_State'], right_on = ['Country.Region','Province.State'], how = 'left')
<filter>
del X_train, y_train
display(covid_pop_sel[covid_pop_sel['Country.Region']=='Italy'])
 display(train_fil[train_fil['Country_Region']=='Italy'] )<feature_engineering>
model.save('./model', save_format="tf")
 print('export saved model.')
<merge>
model = tf.keras.models.load_model('./model' )
<data_type_conversions><EOS> 
THRESHOLD = 0
env = janestreet.make_env()
iter_test = env.iter_test()
print('predicting...')
 for(test_df, pred_df)in tqdm(env.iter_test()):
global pred
if test_df['weight'].item() > 0:
X_test = test_df.loc[:, features].values
if np.isnan(X_test.sum()):
X_test = np.nan_to_num(X_test)+ np.isnan(X_test)* features_mean.values
pred = model(X_test, training = False ).numpy()
pred = np.mean(pred)
 pred_df.action = np.where(pred > THRESHOLD, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df)