Kaggle competition: Jane Street Market Prediction
classes = ["car", "motorcycle", "bus", "bicycle", "truck", "pedestrian", "other_vehicle", "animal", "emergency_vehicle"]
train_dataset = LyftDataset(data_path='.', json_path='.. /input/3d-object-detection-for-autonomous-vehicles/train_data', verbose=True )<create_dataframe>
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('weight>0' ).reset_index(drop=True)
 train.fillna(train.median() ,inplace=True)
 train['feature_stock_id_sum'] = train['feature_41'] + train['feature_42'] + train['feature_43']
train['feature_1_2_cross'] = train['feature_1']/(train['feature_2']+1e-5)
 NUM_TRAIN_EXAMPLES = len(train )
class_heights = {'animal':0.51,'bicycle':1.44,'bus':3.44,'car':1.72,'emergency_vehicle':2.39,'motorcycle':1.59,
'other_vehicle':3.23,'pedestrian':1.78,'truck':3.44}
level5data = LyftDataset(data_path='.', json_path='.. /input/3d-object-detection-for-autonomous-vehicles/test_data', verbose=True )<define_search_space>
features = [c for c in train.columns if 'feature' in c]
f_mean = np.nanmedian(train[features[1:]].values,axis=0 )
voxel_size =(0.4, 0.4, 1.5)
 z_offset = -2.0
bev_shape =(336, 336, 3)
 box_scale = 0.8<create_dataframe>
TRAINING = False
PATH = '.. /input/highwayjs'
records = [(level5data.get('sample', record['first_sample_token'])['timestamp'], record)for record in level5data.scene]
entries = []
for start_time, record in sorted(records):
start_time = level5data.get('sample', record['first_sample_token'])['timestamp'] / 1000000
token = record['token']
name = record['name']
date = datetime.utcfromtimestamp(start_time)
 host = "-".join(record['name'].split("-")[:2])
 first_sample_token = record["first_sample_token"]
entries.append(( host, name, date, token, first_sample_token))
df = pd.DataFrame(entries, columns=["host", "scene_name", "date", "scene_token", "first_sample_token"] )<define_variables>
resp_cols = [c for c in train.columns if 'resp' in c]
test_data_folder = '.. /input/lyft3d-mask-test-data/test_data/test_data'
class BEVImageDataset(torch.utils.data.Dataset):
def __init__(self, sample_token,test_data_folder):
self.sample_token = sample_token
self.test_data_folder = test_data_folder
def __len__(self):
return len(self.sample_token)
 def __getitem__(self, idx):
sample_token = self.sample_token[idx]
input_filepath = os.path.join(test_data_folder,f"{sample_token}_input.png")
 map_filepath = os.path.join(test_data_folder,f"{sample_token}_map.png")
 im = cv2.imread(input_filepath, cv2.IMREAD_UNCHANGED)
 map_im = cv2.imread(map_filepath, cv2.IMREAD_UNCHANGED)
 im = np.concatenate(( im, map_im), axis=2)
 im = im.astype(np.float32)/255
im = torch.from_numpy(im.transpose(2,0,1))
return im, sample_token
test_dataset = BEVImageDataset(all_sample_tokens,test_data_folder )<set_options>
def mish(x):
return tf.keras.layers.Lambda(lambda x: x*K.tanh(K.softplus(x)) )(x)
 tf.keras.utils.get_custom_objects().update({'mish': tf.keras.layers.Activation(mish)})
 def create_model(input_shape):
inp = tf.keras.layers.Input(input_shape)
 tmp = tf.keras.layers.BatchNormalization()(inp)
 xs = [tmp]
for _ in range(10):
if len(xs)> 1:
tmp = tf.keras.layers.Concatenate(axis=-1 )(xs)
 else:
tmp = xs[0]
tmp = tf.keras.layers.Dense(32,activation='mish' )(tmp)
 tmp = tf.keras.layers.BatchNormalization()(tmp)
 tmp = tf.keras.layers.Dropout(0.2 )(tmp)
 xs.append(tmp)
 output = tf.keras.layers.Dense(len(resp_cols),activation='sigmoid' )(tf.keras.layers.Concatenate()(xs))
model = tf.keras.models.Model(inp,output)
 optimizer = tfa.optimizers.RectifiedAdam(1e-3)
 model.compile(optimizer, loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.001),
metrics=[tf.keras.metrics.BinaryAccuracy(name='binary_accuracy')])
 return model
class UNet(nn.Module):
def __init__(
 self,
in_channels=1,
n_classes=2,
depth=5,
wf=6,
padding=False,
batch_norm=False,
up_mode='upconv', 
):

 super(UNet, self ).__init__()
assert up_mode in('upconv', 'upsample')
 self.padding = padding
self.depth = depth
prev_channels = in_channels
self.down_path = nn.ModuleList()
for i in range(depth):
self.down_path.append(
 UNetConvBlock(prev_channels, 2 **(wf + i), padding, batch_norm)
)
 prev_channels = 2 **(wf + i)
 self.up_path = nn.ModuleList()
for i in reversed(range(depth - 1)) :
self.up_path.append(
 UNetUpBlock(prev_channels, 2 **(wf + i), up_mode, padding, batch_norm)
)
 prev_channels = 2 **(wf + i)
 self.last = nn.Conv2d(prev_channels, n_classes, kernel_size=1)
 def forward(self, x):
blocks = []
for i, down in enumerate(self.down_path):
x = down(x)
 if i != len(self.down_path)- 1:
blocks.append(x)
 x = F.max_pool2d(x, 2)
 for i, up in enumerate(self.up_path):
x = up(x, blocks[-i - 1])
 return self.last(x)
 class UNetConvBlock(nn.Module):
def __init__(self, in_size, out_size, padding, batch_norm):
super(UNetConvBlock, self ).__init__()
block = []
block.append(nn.Conv2d(in_size, out_size, kernel_size=3, padding=int(padding)))
 block.append(nn.ReLU())
 if batch_norm:
block.append(nn.BatchNorm2d(out_size))
block.append(nn.Conv2d(out_size, out_size, kernel_size=3, padding=int(padding)))
 block.append(nn.ReLU())
 if batch_norm:
block.append(nn.BatchNorm2d(out_size))
self.block = nn.Sequential(*block)
 def forward(self, x):
out = self.block(x)
 return out
class UNetUpBlock(nn.Module):
def __init__(self, in_size, out_size, up_mode, padding, batch_norm):
super(UNetUpBlock, self ).__init__()
if up_mode == 'upconv':
self.up = nn.ConvTranspose2d(in_size, out_size, kernel_size=2, stride=2)
 elif up_mode == 'upsample':
self.up = nn.Sequential(
 nn.Upsample(mode='bilinear', scale_factor=2),
nn.Conv2d(in_size, out_size, kernel_size=1), 
)
 self.conv_block = UNetConvBlock(in_size, out_size, padding, batch_norm)
 def center_crop(self, layer, target_size):
_, _, layer_height, layer_width = layer.size()
diff_y =(layer_height - target_size[0])// 2
diff_x =(layer_width - target_size[1])// 2
return layer[
:, :, diff_y :(diff_y + target_size[0]), diff_x :(diff_x + target_size[1])
 ]
def forward(self, x, bridge):
up = self.up(x)
 crop1 = self.center_crop(bridge, up.shape[2:])
 out = torch.cat([up, crop1], 1)
 out = self.conv_block(out)
 return out<choose_model_class>
def set_all_seeds(seed):
np.random.seed(seed)
 random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed )
def get_unet_model(in_channels=6, num_output_classes=2):
model = UNet(in_channels=in_channels, n_classes=num_output_classes, wf=5, depth=4, padding=True, up_mode='upsample')
 model = nn.DataParallel(model)
 return model 
<categorify>
X_tr = train.query('date<440')[features].values
y_tr =(train.query('date<440')[resp_cols].values > 0 ).astype(int)
 X_val = train.query('date>460')[features].values
y_val =(train.query('date>460')[resp_cols].values > 0 ).astype(int)
 if TRAINING:
metric = {}
for seed in [6, 28, 496, 8128]:
set_all_seeds(seed)
 model = create_model(X_tr.shape[-1])
 hist = model.fit(X_tr,y_tr,
validation_data=(X_val,y_val),
epochs=200,
batch_size=8192,
callbacks=[tf.keras.callbacks.EarlyStopping('val_binary_accuracy',mode='max',patience=20)])
 model.save_weights(f'model_{seed}.tf')
 metric[seed] = max(hist.history['binary_accuracy'])
 print(metric)
 else:
models = []
for seed in [6, 28, 496, 8128]:
model = create_model(X_tr.shape[-1])
 model.load_weights(f'{PATH}/model_{seed}.tf')
 model.call = tf.function(model.call, experimental_relax_shapes=True)
 models.append(model)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 class_weights = torch.from_numpy(np.array([0.2] + [1.0]*len(classes), dtype=np.float32))
class_weights = class_weights.to(device )<load_pretrained>
blend = np.asarray([0.55722,0.55301,0.56278,0.55624])
 blend = blend/sum(blend )
<normalization><EOS> 
if not TRAINING:
f = np.median
janestreet.competition.make_env.__called__ = False
env = janestreet.make_env()
th = 0.503
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
test_df['feature_stock_id_sum'] = test_df['feature_41'] + test_df['feature_42'] + test_df['feature_43']
test_df['feature_1_2_cross'] = test_df['feature_1']/(test_df['feature_2']+1e-5)
 x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
p = sum(blend*np.asarray([f(model(x_tt,training=False ).numpy())for model in models]))
pred_df.action = np.where(p > th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )