Kaggle competition: Jane Street Market Prediction
df_train = df_union[df_union['DsType']=='train'].drop('DsType',axis=1)
 df_test = df_union[df_union['DsType']=='test'].drop('DsType',axis=1)
 df_train['ConfirmedCasesValid'] = 0
df_train['FatalitiesValid'] = 0
df_train['ConfirmedCasesTrend'] = 0
df_train['FatalitiesTrend'] = 0
df_train['ConfirmedCasesResid'] = 0
df_train['FatalitiesResid'] = 0
df_test['ConfirmedCases'] = 0
df_test['Fatalities'] = 0
df_test['ConfirmedCasesTrend'] = 0
df_test['FatalitiesTrend'] = 0
df_test['ConfirmedCasesResid'] = 0
df_test['FatalitiesResid'] = 0<data_type_conversions>
if tuning or training:
train_data = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 train_data.fillna(train_data.mean() ,inplace=True)
 start_date=86
feature_columns = [col for col in train_data.columns.values if 'feature' in col]
corr=abs(train_data[feature_columns].corr())
 ordered_feature_columns=[feature_columns.pop(0)]
for col in feature_columns:
corr_max = corr[col][ordered_feature_columns].idxmax()
corr_max_idx = ordered_feature_columns.index(corr_max)
 ordered_feature_columns.insert(corr_max_idx+1,col)
 f_mean = train_data[ordered_feature_columns].mean()
f_mean.to_csv('f_mean.csv',index=False)
 with open("ordered_columns.txt", 'wb')as f:
pickle.dump(( ordered_feature_columns), f)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
train_data = train_data[train_data['date']>=start_date]
X = train_data[ordered_feature_columns].values
y = np.stack([(train_data[c] > 0 ).astype('int')for c in resp_cols] ).T
max_date = train_data['date'].max()
split_date = int(0.7*(max_date-start_date))
train_test_split = train_data['date'][train_data['date']==split_date].index[0]
del train_data
X_train=X[:train_test_split,:]
X_test=X[train_test_split:,:]
y_train=y[:train_test_split,:]
y_test=y[train_test_split:,:]
y1_train = df_train['ConfirmedCases'].astype(float)
 y2_train = df_train['Fatalities'].astype(float )<remove_duplicates>
tf.random.set_seed(SEED)
 def create_model(hp, num_columns, num_labels,encoder):
inp = tf.keras.layers.Input(shape =(num_columns, 1))
x1 = encoder(inp)
 x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Conv1D(filters=8,
kernel_size=hp.Int('kernel_size',5,10,step=5),
strides=1,
activation='relu' )(x)
 x = tf.keras.layers.MaxPooling1D(pool_size=2 )(x)
 x = tf.keras.layers.Flatten()(x)
 x = tf.keras.layers.Concatenate()([x,x1])
 for i in range(hp.Int('num_layers',4,6)) :
x = tf.keras.layers.Dense(hp.Int(f'num_units_{i}',256,512))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(hp.Float(f'dropout_{i}',0,0.5))(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = hp.Float('lr',0.00001,0.1,default=0.001)) ,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.01),
metrics = tf.keras.metrics.AUC(name = 'AUC'), 
)
 return model
def create_autoencoder(input_dim,output_dim):
i = tf.keras.layers.Input(input_dim)
 encoded = tf.keras.layers.BatchNormalization()(i)
 encoded = tf.keras.layers.GaussianNoise(0.05 )(encoded)
 encoded = tf.keras.layers.Dense(64,activation='relu' )(encoded)
 decoded = tf.keras.layers.Dropout(0.2 )(encoded)
 decoded = tf.keras.layers.Dense(input_dim,name='decoded' )(decoded)
 x = tf.keras.layers.Dense(32,activation='relu' )(decoded)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = Model(inputs=i,outputs=encoded)
 autoencoder = Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=Adam(0.001),loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder
NPT = 4
md1 = RandomForestRegressor(random_state=1234)
 md2 = RandomForestRegressor(random_state=1234)
 for country in df_train['Country_Region'].unique() :
df_train_cy = df_train[df_train['Country_Region']==country].copy()
df_test_cy = df_test[df_test['Country_Region']==country].copy()
for province in df_train_cy['Province_State'].unique() :
print(f'Analizing > country = {country} : province = {province}')
 df_train_pr = df_train_cy[df_train_cy['Province_State']==province].copy()
df_test_pr = df_test_cy[df_test_cy['Province_State']==province].copy()
X_train_pr = df_train_pr[['DateOrd']]
y1_train_pr = df_train_pr['ConfirmedCases']
y2_train_pr = df_train_pr['Fatalities']
df_test_pr = df_test_pr[df_test_pr['Province_State']==province].copy()
X_test_pr = df_test_pr[['DateOrd']]
y1_check_pr_trend, y1_pred_pr_trend = polynomial_trend(NPT,X_train_pr,y1_train_pr,X_test_pr)
 y2_check_pr_trend, y2_pred_pr_trend = polynomial_trend(NPT,X_train_pr,y2_train_pr,X_test_pr)
 df_train.loc[(( df_train['Country_Region']==country)&(df_train['Province_State']==province)) ,'ConfirmedCasesTrend'] = y1_check_pr_trend
df_train.loc[(( df_train['Country_Region']==country)&(df_train['Province_State']==province)) ,'FatalitiesTrend'] = y2_check_pr_trend
df_test.loc[(( df_test['Country_Region']==country)&(df_test['Province_State']==province)) ,'ConfirmedCasesTrend'] = y1_pred_pr_trend
df_test.loc[(( df_test['Country_Region']==country)&(df_test['Province_State']==province)) ,'FatalitiesTrend'] = y2_pred_pr_trend
y1_train_pr_resid = y1_train_pr - y1_check_pr_trend
y2_train_pr_resid = y2_train_pr - y2_check_pr_trend
md1.fit(X_train_pr,y1_train_pr_resid)
 md2.fit(X_train_pr,y2_train_pr_resid)
 y1_check_pr_resid = md1.predict(X_train_pr)
 y2_check_pr_resid = md2.predict(X_train_pr)
 y1_pred_pr_resid = md1.predict(X_test_pr)
 y2_pred_pr_resid = md2.predict(X_test_pr)
 df_train.loc[(( df_train['Country_Region']==country)&(df_train['Province_State']==province)) ,'ConfirmedCasesResid'] = y1_train_pr_resid
df_train.loc[(( df_train['Country_Region']==country)&(df_train['Province_State']==province)) ,'FatalitiesResid'] = y2_train_pr_resid
df_test.loc[(( df_test['Country_Region']==country)&(df_test['Province_State']==province)) ,'ConfirmedCasesResid'] = y1_pred_pr_resid
df_test.loc[(( df_test['Country_Region']==country)&(df_test['Province_State']==province)) ,'FatalitiesResid'] = y2_pred_pr_resid
df_train.loc[(( df_train['Country_Region']==country)&(df_train['Province_State']==province)) ,'ConfirmedCasesValid'] = y1_check_pr_trend + y1_check_pr_resid
df_train.loc[(( df_train['Country_Region']==country)&(df_train['Province_State']==province)) ,'FatalitiesValid'] = y2_check_pr_trend + y2_check_pr_resid
df_test.loc[(( df_test['Country_Region']==country)&(df_test['Province_State']==province)) ,'ConfirmedCases'] = y1_pred_pr_trend + y1_pred_pr_resid
df_test.loc[(( df_test['Country_Region']==country)&(df_test['Province_State']==province)) ,'Fatalities'] = y2_pred_pr_trend + y2_pred_pr_resid<import_modules>
autoencoder, encoder = create_autoencoder(130,5)
 if training:
autoencoder.fit(X,(X,y),
epochs=1000,
batch_size=4096,
validation_split=0.1,
callbacks=[EarlyStopping('val_loss',patience=10,restore_best_weights=True)])
 encoder.save_weights('JS_CNN_encoder.hdf5')
 else:
encoder.load_weights('/kaggle/input/jscnn/JS_CNN_encoder_seed_111.hdf5')
 encoder.trainable = False
from sklearn.metrics import mean_squared_log_error<prepare_x_and_y>
if tuning:
model_fn = lambda hp: create_model(hp,X_train.shape[-1],y_train.shape[-1],encoder)
 tuner = kt.tuners.bayesian.BayesianOptimization(
 hypermodel=model_fn,
objective= kt.Objective('val_AUC', direction='max'),
num_initial_points=4,
max_trials=20)
 tuner.search(X_train,y_train,batch_size=4096,epochs=20, validation_data =(X_test, y_test), callbacks=[EarlyStopping('val_AUC', mode='max',patience=5)])
 hp = tuner.get_best_hyperparameters(1)[0]
pd.to_pickle(hp,'best_hp_cnn_day_86_encoder_seed_111.pkl' )
def plotExample() :
train_cond =(( df_train['Country_Region']==my_country)&(df_train['Province_State']==my_province))
test_cond =(( df_test['Country_Region']==my_country)&(df_test['Province_State']==my_province))
x_test_plt = df_test[test_cond]['Date']
y_test_plt = df_test[test_cond][my_variable]
x_train_plt = df_train[train_cond]['Date']
y_train_plt = df_train[train_cond][my_variable]
y_valid_plt = df_train[train_cond][my_variable+'Valid']
plt.rcParams["figure.figsize"] =(12,6)
 fig, ax = plt.subplots()
ax.plot(x_train_plt,y_train_plt,'o')
 ax.plot(x_train_plt,y_valid_plt,'x')
 ax.plot(x_test_plt,y_test_plt,'*')
 ax.set_xticks([] )<define_variables>
if training:
hp = pd.read_pickle('best_hp_cnn_day_86_encoder_seed_111.pkl')
 model_fn = lambda hp: create_model(hp,X_train.shape[-1],y_train.shape[-1],encoder)
 model = model_fn(hp)
 model.fit(X_train,y_train,validation_data=(X_test,y_test),epochs=100,batch_size=4096,
callbacks=[EarlyStopping('val_AUC',mode='max',patience=10,restore_best_weights=True)])
 model.save_weights('JS_CNN_day_86_encoder_seed_111.hdf5' )
my_country='Italy'
my_province='ND'
my_variable='ConfirmedCases'
plotExample()<prepare_x_and_y>
if not training or tuning:
model_fn = lambda hp: create_model(hp,130,5,encoder)
 hp = pd.read_pickle('/kaggle/input/jscnn/best_hp_cnn_day_86_encoder_seed_111.pkl')
 model = model_fn(hp)
 model.load_weights('/kaggle/input/jscnn/JS_CNN_day_86_encoder_seed_111.hdf5')
 samples_mean = pd.read_csv('/kaggle/input/jscnn/f_mean.csv')
 env = janestreet.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in iter_test:
weight = test_df.weight.iloc[0]
with open("/kaggle/input/jscnn/ordered_columns.txt", 'rb')as f:
ordered_cols = pickle.load(f)
 test_df = test_df[ordered_cols]
X_test = test_df.values[0]
if weight==0:
sample_prediction_df.action = 0
else:
for index, x in np.ndenumerate(X_test):
idx=index[0]
if np.isnan(x):
X_test[idx] = samples_mean.iloc[idx]
X=X_test.reshape(( 1,-1))
prediction = np.median(model(X, training=False ).numpy() [0])
 sample_prediction_df.action = int(round(prediction))
env.predict(sample_prediction_df)