Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<prepare_x_and_y>
%%time
np.random.seed(777)
 fake_valid_inds = np.random.choice(X_train.index.values, 2_000_000, replace = False)
 train_inds = np.setdiff1d(X_train.index.values, fake_valid_inds)
 train_data = lgb.Dataset(X_train.drop(columns=['id'] ).loc[train_inds] , label = y_train.loc[train_inds],
categorical_feature=id_encodings, free_raw_data=False)
 fake_valid_data = lgb.Dataset(X_train.drop(columns=['id'] ).loc[fake_valid_inds], label = y_train.loc[fake_valid_inds],
categorical_feature=id_encodings,
free_raw_data=False )<train_model>
start_time = time.time()
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 print('Load in data successful!')
 print('-----------------------------------------------------------------------')
 time_passed_m = int(( time.time() - start_time)// 60)
 time_passed_s = int(( time.time() - start_time)% 60)
 print(f'Time usage: {time_passed_m}m.{time_passed_s}s.' )
%%time
m_lgb = lgb.train(best_params, train_data, valid_sets = [fake_valid_data], verbose_eval=1 )<predict_on_test>
SEED = 42
np.random.seed(SEED)
 TRAINING_PGTS = True
MANUAL_VALIDATING = False
TRAINING = True
print(f'TRAINING_PGTS = {TRAINING_PGTS}
\
MANUAL_VALIDATING = {MANUAL_VALIDATING}
\
TRAINING = {TRAINING}' )
for d in range(1886, 1914):
df['F_' + str(d)] = m_lgb.predict(X_val[X_val['day']==d].drop(columns=['id']))<compute_test_metric>
train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() , inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if 'feature' in c]
f_mean = np.mean(train[features[1:]].values, axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
print(X_train.shape, y_train.shape)
 if MANUAL_VALIDATING:
X_train_v = train[train['date'] <= 416]
y_train_v = np.stack([(X_train_v[c] > 0 ).astype('int')for c in resp_cols] ).T
X_train_v = X_train_v.loc[:, features].values
X_test_v = train[train['date'] > 416]
y_test_v = np.stack([(X_test_v[c] > 0 ).astype('int')for c in resp_cols] ).T
X_test_v = X_test_v.loc[:, features].values
print(X_test_v.shape, y_test_v.shape)
 print('Done data preprocessing!' )
h = 28
n = 1885
def rmsse(ground_truth, forecast, train_series, axis=1):
assert axis == 0 or axis == 1
assert type(ground_truth)== np.ndarray and type(forecast)== np.ndarray and type(train_series)== np.ndarray
if axis == 1:
assert ground_truth.shape[1] > 1 and forecast.shape[1] > 1 and train_series.shape[1] > 1
numerator =(( ground_truth - forecast)**2 ).sum(axis=axis)
 if axis == 1:
denominator = 1/(n-1)*(( train_series[:, 1:] - train_series[:, :-1])** 2 ).sum(axis=axis)
 else:
denominator = 1/(n-1)*(( train_series[1:] - train_series[:-1])** 2 ).sum(axis=axis)
 if(numerator < 0 ).any() :
print('nu')
 elif(denominator < 0 ).any() :
print(denominator[denominator < 0])
 return(1/h * numerator/denominator)** 0.5<define_variables>
def create_mlp(num_columns, num_labels, hidden_units,
dropout_rates, label_smoothing, learning_rate):
inp = Input(shape=(num_columns,))
x = BatchNormalization()(inp)
 x = Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = Dense(hidden_units[i] )(x)
 x = BatchNormalization()(x)
 x = Activation(tf.keras.activations.swish )(x)
 x = Dropout(dropout_rates[i+1] )(x)
 x = Dense(num_labels )(x)
 out = Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer = tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics = tf.keras.metrics.AUC(name='AUC')
)
 return model
level_groupings = {2: ["state_id"], 3: ["store_id"], 4: ["cat_id"], 5: ["dept_id"],
6: ["state_id", "cat_id"], 7: ["state_id", "dept_id"], 8: ["store_id", "cat_id"], 9: ["store_id", "dept_id"],
10: ["item_id"], 11: ["item_id", "state_id"]}<feature_engineering>
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]
new_agg_df = pd.DataFrame(df[[c for c in df.columns if c.find("d_")== 0 or c.find("F_")== 0]].sum() ).transpose()
new_agg_df["level"] = 1
new_agg_df["weight"] = 1/12
column_order = new_agg_df.columns
for level in level_groupings:
temp_df = df.groupby(by=level_groupings[level] ).sum().reset_index()
temp_df["level"] = level
new_agg_df = new_agg_df.append(temp_df[column_order])
 del temp_df
agg_df = new_agg_df
train_series_cols = [c for c in df.columns if c.find("d_")== 0 and int(c.split('_')[1])< 1886]
ground_truth_cols = [c for c in df.columns if c.find("d_")== 0 and int(c.split('_')[1])in range(1886, 1914)]
forecast_cols = [c for c in df.columns if c.find("F_")== 0]
df["rmsse"] = rmsse(np.array(df[ground_truth_cols]),
np.array(df[forecast_cols]), np.array(df[train_series_cols]))
agg_df["rmsse"] = rmsse(np.array(agg_df[ground_truth_cols]),
np.array(agg_df[forecast_cols]), np.array(agg_df[train_series_cols]))
df["wrmsse"] = df["weight"] * df["rmsse"]
agg_df["wrmsse"] = agg_df["weight"] * agg_df["rmsse"]
print(df["wrmsse"].sum() + agg_df["wrmsse"].sum() )<predict_on_test>
n_samples = 2000
n_groups = 20
assert n_samples % n_groups == 0
idx = np.linspace(0, n_samples-1, num=n_samples)
 X_train_pgts = np.random.random(size=(n_samples, 5))
y_train_pgts = np.random.choice([0, 1], n_samples)
 groups = np.repeat(np.linspace(0, n_groups-1, num=n_groups), n_samples/n_groups)
 groups.shape
submit_df = df[["id"]]
for i in range(1, 29):
submit_df["F" + str(i)] = m_lgb.predict(X_test[X_test['day']==i+1913].drop(columns=['id']))<prepare_output>
FOLDS = 5
models = []
if TRAINING_PGTS:
gkf = PurgedGroupTimeSeriesSplit(n_splits=FOLDS, group_gap=20)
 splits = list(gkf.split(y_train, groups=train['date'].values))
for fold,(train_indices, test_indices)in tqdm(enumerate(splits)) :
X_train_pgts, X_test_pgts = X_train.iloc[train_indices, :], X_train.iloc[test_indices, :]
y_train_pgts, y_test_pgts = y_train[train_indices], y_train[test_indices]
tf.keras.backend.clear_session()
model = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 er = EarlyStopping(patience = 8,
restore_best_weights = True,
monitor = 'val_loss')
 ReduceLR = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss',
factor = 0.1,
patience = 8,
verbose = 1,
mode = 'min')
 model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
 filepath = f'model3-1_{SEED}_{fold}.hdf5',
save_weights_only = True,
verbose = 0,
monitor = 'val_loss',
save_best_only = True)
 nn_callbacks = [er, ReduceLR, model_checkpoint_callback]
model.fit(X_train_pgts, y_train_pgts,
validation_data =(X_test_pgts, y_test_pgts),
epochs = epochs[0],
batch_size = batch_size[1],
verbose=2,
callbacks = nn_callbacks)
 models.append(model)
 else:
for fold in tqdm(range(FOLDS)) :
tf.keras.backend.clear_session()
model = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 model.load_weights(f'.. /input/js-model-v1/model4_{SEED}_{fold}.hdf5')
 models.append(model )
submit_df2 = submit_df.copy()
submit_df2["id"] = submit_df2["id"].apply(lambda x: x.replace('validation',
'evaluation'))<concatenate>
if MANUAL_VALIDATING:
er = EarlyStopping(patience = 8,
restore_best_weights = True,
monitor = 'val_loss')
 ReduceLR = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss',
factor = 0.1,
patience = 8,
verbose = 1,
mode = 'min')
 model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
 filepath = 'js_model_v4_weights.h5',
save_weights_only = True,
verbose = 0,
monitor = 'val_loss',
save_best_only = True)
 nn_callbacks = [er, ReduceLR, model_checkpoint_callback]
clf = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 history = clf.fit(
 X_train_v, y_train_v,
epochs = epochs[0],
batch_size = batch_size,
verbose = 2,
validation_data =(X_test_v, y_test_v),
callbacks = nn_callbacks 
)
 del X_train_v, y_train_v, X_test_v, y_test_v
if TRAINING:
tf.keras.backend.clear_session()
tf.random.set_seed(SEED)
 clf = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 clf.fit(X_train, y_train, epochs=epochs[0],
batch_size=batch_size[1], verbose=2, 
)
 clf.save_weights('js_model_v2-1_weights.h5')
 print('Training succeeded!
')
 else:
clf = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 clf.load_weights('.. /input/js-model-v1/model3-1_1111_9.hdf5')
 print('Loading succeeded!
')
 clf.summary()
models.append(clf )