Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<define_variables>
tf.config.optimizer.set_jit(True)
 SEED = 0
train_mode = True
tf.random.set_seed(SEED)
 np.random.seed(SEED )
df_train = df_train_co_pop
df_test = df_test_co_pop<categorify>
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop=True)
 train = train[train['weight'] != 0].reset_index(drop=True)
 train.fillna(train.mean() , inplace=True)
 train = reduce_mem_usage(train)
 features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[features[1:]].values, axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
resp_target = ['resp']
le = preprocessing.LabelEncoder()
df_train.Country = le.fit_transform(df_train.Country)
 df_train['State'] = le.fit_transform(df_train['State'])
 df_train.Country = le.fit_transform(df_train.Country)
 df_test.Country = le.fit_transform(df_test.Country)
 df_test['State'] = le.fit_transform(df_test['State'])
 df_test.head()<categorify>
class UtilityScoreCallback(tf.keras.callbacks.Callback):
def __init__(self, X, date, weight, resp, batch_size = 1024,
early_stopping_patience = 30, plateau_patience = 10, min_lr = 1e-6,
reduction_rate = 0.3, stage = 'train', fold_n = 0, verbose = 1):
super(Callback, self ).__init__()
self.X = X
self.date = date
self.weight = weight
self.resp = resp
self.history = []
self.batch_size = batch_size
self.early_stopping_patience = early_stopping_patience
self.plateau_patience = plateau_patience
self.min_lr = min_lr
self.reduction_rate = reduction_rate
self.fold_n = fold_n
self.stage = stage
self.verbose = verbose
self.best_uscore = - float('inf')
 self.checkpoints_path = f'models_1.0/'
if not os.path.exists(self.checkpoints_path):
os.makedirs(self.checkpoints_path)
 def utility_score(self, d, w, r, a):
Pi = np.bincount(d, w * r * a)
 t = np.sum(Pi)/ np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / len(Pi))
u = min(max(t, 0), 6)* np.sum(Pi)
 return u
def is_patience_lost(self, patience):
if len(self.history)> patience:
best_performance = max(self.history[-(patience + 1):-1])
 return best_performance == self.history[-(patience + 1)] and best_performance >= self.history[-1]
def early_stopping_check(self, uscore):
if self.is_patience_lost(self.early_stopping_patience):
self.model.stop_training = True
def model_checkpoint(self, uscore, epoch):
if uscore > self.best_uscore:
for checkpoint in glob.glob(os.path.join(self.checkpoints_path, f'JSModel_{self.fold_n}*')) :
os.remove(checkpoint)
 self.best_uscore = uscore
self.model.save(os.path.join(self.checkpoints_path, f'ResnetModel_{self.fold_n}.hdf5'))
if self.verbose:
print(f"
{'
Saved new checkpoint with Score {self.best_uscore}
{'
")
 def reduce_lr_on_plateau(self):
if self.is_patience_lost(self.plateau_patience):
new_lr = max(float(K.get_value(self.model.optimizer.lr)) * self.reduction_rate, self.min_lr)
 K.set_value(self.model.optimizer.lr, new_lr)
 if self.verbose:
print(f"
{'
Reduced learning rate to {new_lr}.
 {'
")
 def on_epoch_end(self, epoch, logs={}):
y_preds = self.model.predict(self.X, batch_size=self.batch_size)
 action_proba = np.median(y_preds, axis=1)
 action = np.where(action_proba >= 0.5, 1, 0)
 uscore = self.utility_score(self.date, self.weight, self.resp, action)
 self.history.append(uscore)
 if self.stage == 'val':
self.early_stopping_check(uscore)
 self.model_checkpoint(uscore, epoch)
 if self.plateau_patience >= 0:
self.reduce_lr_on_plateau()
def get_uscore_history(self):
return self.history
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
countries = df_train.Country.unique()<prepare_x_and_y>
def create_resnet(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x_1 = tf.keras.layers.Concatenate(axis=-1 )([x, inp])
 for i in range(len(hidden_units)) :
x_1 = tf.keras.layers.Dense(hidden_units[i] )(x_1)
 x_1 = tf.keras.layers.BatchNormalization()(x_1)
 x_1 = tf.keras.layers.Activation(tf.keras.activations.swish )(x_1)
 x_1 = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x_1)
 x_2 = tf.keras.layers.Concatenate(axis=-1 )([x_1, x])
 for i in range(len(hidden_units)) :
x_2 = tf.keras.layers.Dense(hidden_units[i] )(x_2)
 x_2 = tf.keras.layers.BatchNormalization()(x_2)
 x_2 = tf.keras.layers.Activation(tf.keras.activations.swish )(x_2)
 x_2 = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x_2)
 x_3 = tf.keras.layers.Concatenate(axis=-1 )([x_1, x_2])
 x_3 = tf.keras.layers.Dense(256 )(x_3)
 x_3 = tf.keras.layers.BatchNormalization()(x_3)
 x_3 = tf.keras.layers.Activation(tf.keras.activations.swish )(x_3)
 x_3 = tf.keras.layers.Dropout(0.2 )(x_3)
 x_3 = tf.keras.layers.Dense(num_labels )(x_3)
 out = tf.keras.layers.Activation("sigmoid" )(x_3)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
xout = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = df_train.loc[df_train.Country == country, :].State.unique()
for state in states:
x_train_CS = df_train.loc[(df_train.Country == country)&(df_train.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities','Population']]
y1_train_CS = x_train_CS.loc[:, 'ConfirmedCases']
y2_train_CS = x_train_CS.loc[:, 'Fatalities']
x_train_CS = x_train_CS.loc[:, ['State', 'Country', 'Date','Population']]
x_train_CS.Country = le.fit_transform(x_train_CS.Country)
 x_train_CS['State'] = le.fit_transform(x_train_CS['State'])
 x_test_CS = df_test.loc[(df_test.Country == country)&(df_test.State == state), ['State', 'Country', 'Date', 'ForecastId','Population']]
x_test_CS_Id = x_test_CS.loc[:, 'ForecastId']
x_test_CS = x_test_CS.loc[:, ['State', 'Country', 'Date','Population']]
x_test_CS.Country = le.fit_transform(x_test_CS.Country)
 x_test_CS['State'] = le.fit_transform(x_test_CS['State'])
 xmodel1 = XGBRegressor(n_estimators=1000)
 xmodel1.fit(x_train_CS, y1_train_CS)
 y1_xpred = xmodel1.predict(x_test_CS)
 xmodel2 = XGBRegressor(n_estimators=1000)
 xmodel2.fit(x_train_CS, y2_train_CS)
 y2_xpred = xmodel2.predict(x_test_CS)
 xdata = pd.DataFrame({'ForecastId': x_test_CS_Id, 'ConfirmedCases': y1_xpred, 'Fatalities': y2_xpred})
 xout = pd.concat([xout, xdata], axis=0)
<feature_engineering>
batch_size = 2048
hidden_units = [150, 150, 150]
dropout_rates = [0.25, 0.25, 0.25, 0.25]
label_smoothing = 1e-3
learning_rate = 1e-3
folds = 5
train_mode = True
opt_th_cross = 0.5
xout.ForecastId = xout.ForecastId.astype('int')
 xout['ConfirmedCases'] = round(xout['ConfirmedCases'],1)
 xout['Fatalities'] = round(xout['Fatalities'],1)
 display(xout.head() )<data_type_conversions>
if train_mode:
clf = create_resnet(len(features), 5, hidden_units, dropout_rates, label_smoothing, learning_rate)
 clf.fit(train.loc[:, features].values,(train.loc[:,resp_cols] > 0 ).astype(int), epochs=150, batch_size=batch_size, shuffle=True)
xout['ConfirmedCases'] = xout['ConfirmedCases'].apply(int)
 xout['Fatalities'] = xout['Fatalities'].apply(int )<remove_duplicates>
models = []
clf.call = tf.function(clf.call, experimental_relax_shapes=True)
 models.append(clf )
xout = xout.drop_duplicates()
xout.reindex()<save_to_csv>
env = janestreet.make_env()
env_iter = env.iter_test()
xout.to_csv('submission.csv', index=False )<import_modules>
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].values[0] > 0:
test_df = test_df.loc[:, features].values
if np.isnan(test_df[:, 1:].sum()):
test_df[:, 1:] = np.nan_to_num(test_df[:, 1:])+ np.isnan(test_df[:, 1:])* f_mean
pred = np.mean([model(test_df, training = False ).numpy() for model in models],axis=0)
 pred_df.action = int(0.6*pred[:,3] + 0.2*pred[:,2] + 0.2*pred[:,4]> opt_th_cross)
 else:
pred_df["action"].values[0] = 0
env.predict(pred_df )