Kaggle competition: Jane Street Market Prediction
<data_type_conversions>
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv' )
df_train_pop['Province.State'].fillna("None", inplace=True)
 df_train_pop['Country.Region'].fillna("None", inplace=True)
 df_train_pop['Population'] = df_train_pop['Population'].fillna(0 )<create_dataframe>
train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
features_mean = []
features = [c for c in train.columns if 'feature' in c]
for i in features:
x = train[i].mean()
features_mean.append(x)
 train[i] = train[i].fillna(x)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
missings_count = {col:df_train_pop[col].isnull().sum() for col in df_train_pop.columns}
missings = pd.DataFrame.from_dict(missings_count, orient='index')
 print(missings.nlargest(30, 0))<create_dataframe>
f = np.median
f_mean = np.mean(train[features[1:]].values,axis=0 )
df_fin_selc = df_train_pop[['Province_State','Country_Region','Date','ConfirmedCases','Fatalities','Day_num','Day','Month','Year','Population']]<feature_engineering>
del train
df_fin_selc['I'] = df_fin_selc['ConfirmedCases']-df_fin_selc['Fatalities']
df_fin_selc['R'] = df_fin_selc['Fatalities']
df_fin_selc['S'] = df_fin_selc['Population']-df_fin_selc['ConfirmedCases']
df_SIR = df_fin_selc[['Province_State','Country_Region','Population','Day_num','S','I','R']].reindex()
df_SIR[df_SIR['Country_Region'] == 'Italy'].tail(5 )<import_modules>
epochs = 200
batch_size = 4096
hidden_units = [160, 160]
dropout_rates = [0.20, 0.20, 0.20]
label_smoothing = 1e-2
learning_rate = 1e-3
import numpy as np
from scipy.integrate import odeint
from scipy import integrate, optimize<compute_test_metric>
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer = tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics = tf.keras.metrics.AUC(name='AUC'))
return model
def SIR_testmodel(y,t,bta,gmma):
S, I, R = y
dS_dt = -1*bta*I*S/N
dI_dt =(bta*I*S/N)- gmma*I
dR_dt = gmma*I
return([dS_dt, dI_dt, dR_dt] )<define_search_space>
clf.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, verbose=2 )
N = 1
S00 = 0.9
I00 = 0.1
R00 = 0.0
bta = 0.35
gmma = 0.1
t = np.linspace(0,100,1000)
 sol = odeint(SIR_testmodel,[S00,I00,R00],t,args =(bta,gmma))
sol = np.array(sol )<filter>
del X_train
del y_train
display(covid_pop_sel[covid_pop_sel['Country.Region']=='Italy'] )<filter>
models = []
models.append(clf)
 th = 0.503
<split><EOS> 
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )