Kaggle competition: Jane Street Market Prediction
<SOS> metric: JaneStreetPnl Kaggle data source: jane-street-market-prediction<set_options>
SEED = 1111
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 train = pd.read_feather('.. /input/janestreet-save-as-feather/train.feather')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
os.environ["OMP_NUM_THREADS"] = "1"
%matplotlib inline 
<categorify>
features1 = [f'feature_{i}' for i in range(71)]
features2 = [f'feature_{i}' for i in range(71, 130)]
class LyftTestDataset(LyftDataset):

 def __init__(self, data_path: str, json_path: str, verbose: bool = True, map_resolution: float = 0.1):

 self.data_path = Path(data_path ).expanduser().absolute()
self.json_path = Path(json_path)
 self.table_names = [
"category",
"attribute",
"sensor",
"calibrated_sensor",
"ego_pose",
"log",
"scene",
"sample",
"sample_data",
"map",
]
start_time = time.time()
self.category = self.__load_table__("category")
 self.attribute = self.__load_table__("attribute")
 self.sensor = self.__load_table__("sensor")
 self.calibrated_sensor = self.__load_table__("calibrated_sensor")
 self.ego_pose = self.__load_table__("ego_pose")
 self.log = self.__load_table__("log")
 self.scene = self.__load_table__("scene")
 self.sample = self.__load_table__("sample")
 self.sample_data = self.__load_table__("sample_data")
 self.map = self.__load_table__("map")
 if verbose:
for table in self.table_names:
print("{} {},".format(len(getattr(self, table)) , table))
print("Done loading in {:.1f} seconds.
 ======".format(time.time() - start_time))
self.explorer = LyftDatasetExplorer(self)
 self.__make_reverse_index__(verbose)
 def __make_reverse_index__(self, verbose: bool)-> None:

 start_time = time.time()
if verbose:
print("Reverse indexing...")
 self._token2ind = dict()
for table in self.table_names:
self._token2ind[table] = dict()
for ind, member in enumerate(getattr(self, table)) :
self._token2ind[table][member["token"]] = ind
for record in self.sample_data:
cs_record = self.get("calibrated_sensor", record["calibrated_sensor_token"])
 sensor_record = self.get("sensor", cs_record["sensor_token"])
 record["sensor_modality"] = sensor_record["modality"]
record["channel"] = sensor_record["channel"]
for record in self.sample:
record["data"] = {}
record["anns"] = []
for record in self.sample_data:
if record["is_key_frame"]:
sample_record = self.get("sample", record["sample_token"])
 sample_record["data"][record["channel"]] = record["token"]
if verbose:
print("Done reverse indexing in {:.1f} seconds.
 ======".format(time.time() - start_time))<load_from_zip>
X_train = [train.loc[:, features1].values,
train.loc[:, features2].values]
!tar -xf.. /input/lyft3d-test-dataset/lyft3d_bev_test_data.tar.gz<create_dataframe>
assert len(features)== len(features1)+ len(features2 )
classes = ["car", "motorcycle", "bus", "bicycle", "truck", "pedestrian", "other_vehicle", "animal", "emergency_vehicle"]
train_dataset = LyftDataset(data_path='.', json_path='.. /input/3d-object-detection-for-autonomous-vehicles/train_data', verbose=True )<create_dataframe>
def create_resnet(n_features, n_features_2, n_labels, learning_rate=1e-03, label_smoothing=1e-02):
input_1 = tf.keras.layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = tf.keras.layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(256, activation="elu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = tf.keras.layers.Concatenate()([input_2, input_3])
 head_2 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(256, "elu"),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(256, "relu"),
],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = tf.keras.layers.Average()([input_3, input_4])
 head_3 = tf.keras.Sequential([
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(128, kernel_initializer='lecun_normal', activation='selu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = tf.keras.models.Model(inputs = [input_1, input_2], outputs = output)
 opt = tfa.optimizers.RectifiedAdam(learning_rate=learning_rate)
 opt = tfa.optimizers.SWA(opt)
 model.compile(optimizer=opt,
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=['AUC'])
 return model
tf.keras.backend.clear_session()
clf = create_resnet(len(features1), len(features2), len(resp_cols), 1e-03, 1e-02)
 clf.summary()
class_heights = {'animal':0.51,'bicycle':1.44,'bus':3.44,'car':1.72,'emergency_vehicle':2.39,'motorcycle':1.59,
'other_vehicle':3.23,'pedestrian':1.78,'truck':3.44}
level5data = LyftTestDataset(data_path='.', json_path='.. /input/3d-object-detection-for-autonomous-vehicles/test_data', verbose=True )<define_search_space>
clf.fit(X_train, y_train, epochs=200, batch_size=4096 * 16)
 clf.save('resnet.h5')
 models = [clf]
th = 0.501
f = np.median
models = models[-3:]
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model([x_tt[:, :len(features1)], x_tt[:, len(features1):]], training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )