Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<prepare_x_and_y>
warnings.filterwarnings("ignore")
Italy_S = Italy_SIR_df['S']
Italy_I = Italy_SIR_df['I']
Italy_R = Italy_SIR_df['R']
Italy_s = np.array(Italy_S, dtype=float)
 Italy_i = np.array(Italy_I, dtype=float)
 Italy_r = np.array(Italy_R, dtype=float)
 display(Italy_s[9],Italy_i[9], Italy_r[9] )<train_model>
%%time
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns} )
def SIR_model(y,t,beta,gamma):
S, I, R = y
N = 60480000
dS_dt = -1*beta*I*S/N
dI_dt =(beta*I*S/N)- gamma*I
dR_dt = gamma*I
return([dS_dt, dI_dt, dR_dt])
 def fit_odeint(x, beta, gamma):
return integrate.odeint(SIR_model,(S0, I0, R0), x, args=(beta, gamma)) [:,1]<train_model>
train.fillna(train.mean() ,inplace=True )
xdata = Italy_SIR_df.Day_num
ydata = Italy_i
xdata = np.array(xdata, dtype=float)
 S0 = 60479998.0
I0 = 2.0
R0 = 0
y = S0, I0, R0
popt, pcov = optimize.curve_fit(fit_odeint, xdata, ydata)
 fitted = fit_odeint(xdata, *popt )<load_from_csv>
train['action'] =(train['resp'] > 0 ).astype('int' )
comp_df = pd.read_csv('.. /input/corona-virus-report/covid_19_clean_complete.csv')
 comp_df.head(5 )<correct_missing_values>
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']
comp_df['Province/State'].fillna("None", inplace=True)
 comp_df[comp_df.iloc[:,0:].isnull().any(axis = 1)].iloc[:,0:].head()<data_type_conversions>
features_train_data = train.iloc[:,7:137]
comp_df['Date'] = pd.to_datetime(comp_df['Date'] )<merge>
all_drop_cols = set(high_correlations.index.get_level_values(0))
covid_pop_sel.loc[covid_pop_sel['Country.Region']=='Korea, South', 'Country.Region'] = 'South Korea'
comp_df_pop = comp_df.merge(covid_pop_sel, left_on = ['Country/Region','Province/State'], right_on = ['Country.Region','Province.State'], how = 'left')
 comp_df_pop.tail(3 )<create_dataframe>
f_mean = train.loc[:, features].mean()
comp_pop_sel = comp_df_pop[['Province/State','Country/Region','Date','Confirmed','Deaths','Recovered','Population']]<create_dataframe>
VALID_DAYS = 0
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp_4', 'resp']
print('Make NN sample...')
 X_train = train[train['date'] <= 499-VALID_DAYS]
y_train = np.stack([(X_train[c]>0 ).astype('int')for c in resp_cols] ).T
X_train = X_train.loc[:, features].values
print(X_train.shape, y_train.shape)
 X_valid = train[train['date'] > 499-VALID_DAYS]
y_valid = np.stack([(X_valid[c]>0 ).astype('int')for c in resp_cols] ).T
X_valid = X_valid.loc[:, features].values
print(X_valid.shape, y_valid.shape )
missings_count = {col:comp_pop_sel[col].isnull().sum() for col in comp_pop_sel.columns}
missings = pd.DataFrame.from_dict(missings_count, orient='index')
 print(missings.nlargest(30, 0))<data_type_conversions>
HIDDEN_LAYER_1 = [256, 256]
HIDDEN_LAYER_2 = [160, 160, 160]
HIDDEN_LAYER_3 = [128, 128, 128, 128]
TARGET_NUM = 5
input = tf.keras.layers.Input(shape=(X_train.shape[1],))
x1 = tf.keras.layers.BatchNormalization()(input)
 x1 = tf.keras.layers.Dropout(0.25 )(x1)
 for units in HIDDEN_LAYER_1:
x1 = tf.keras.layers.Dense(units )(x1)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.Activation(tf.keras.activations.swish )(x1)
 x1 = tf.keras.layers.Dropout(0.25 )(x1)
 x2 = tf.keras.layers.BatchNormalization()(input)
 x2 = tf.keras.layers.Dropout(0.25 )(x2)
 for units in HIDDEN_LAYER_2:
x2 = tf.keras.layers.Dense(units )(x2)
 x2 = tf.keras.layers.BatchNormalization()(x2)
 x2 = tf.keras.layers.Activation(tf.keras.activations.swish )(x2)
 x2 = tf.keras.layers.Dropout(0.25 )(x2)
 x3 = tf.keras.layers.BatchNormalization()(input)
 x3 = tf.keras.layers.Dropout(0.25 )(x3)
 for units in HIDDEN_LAYER_3:
x3 = tf.keras.layers.Dense(units )(x3)
 x3 = tf.keras.layers.BatchNormalization()(x3)
 x3 = tf.keras.layers.Activation(tf.keras.activations.swish )(x3)
 x3 = tf.keras.layers.Dropout(0.25 )(x3)
 x = tf.keras.layers.concatenate([x1, x2, x3])
 x = tf.keras.layers.Dense(TARGET_NUM )(x)
 output = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=input, outputs=output)
 model.compile(
 optimizer = tfa.optimizers.RectifiedAdam(learning_rate=1e-3),
metrics = tf.keras.metrics.AUC(name="AUC"),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-2),
)
comp_pop_sel['Population'] = comp_pop_sel['Population'].fillna(0 )<filter>
history = model.fit(
 x = X_train,
y = y_train,
epochs=25,
batch_size=4096,
validation_data=(X_valid, y_valid), 
)
 models = []
models.append(model )
<categorify><EOS> 
THRESHOLD = 0.502
janestreet.make_env.__called__ = False
env = janestreet.make_env()
print('predicting...')
 for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
X_test = test_df.loc[:, features].values
if np.isnan(X_test.sum()):
X_test = np.nan_to_num(X_test)+ np.isnan(X_test)* f_mean.values
pred = model(X_test, training = False ).numpy()
pred = np.mean(pred)
 pred_df.action = np.where(pred >= THRESHOLD, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )