Kaggle competition: Jane Street Market Prediction
voxel_size =(0.4, 0.4, 1.5)
 z_offset = -2.0
bev_shape =(336, 336, 3)
 box_scale = 0.8<load_pretrained>
import random
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.layers.experimental.preprocessing import Normalization
import tensorflow as tf
import numpy as np
import pandas as pd
from tqdm import tqdm
from random import choices
class BEVImageDataset(torch.utils.data.Dataset):
def __init__(self, input_filepaths, map_filepaths=None):
self.input_filepaths = input_filepaths
def __len__(self):
return len(self.input_filepaths)
 def __getitem__(self, idx):
input_filepath = self.input_filepaths[idx]
sample_token = input_filepath.split("/")[-1].replace("_input.png","")
 im = cv2.imread(input_filepath, cv2.IMREAD_UNCHANGED)
 im = im.astype(np.float32)/255
im = torch.from_numpy(im.transpose(2,0,1))
return im, sample_token
test_data_folder = './artifacts/'
input_filepaths = sorted(glob.glob(os.path.join(test_data_folder, "*_input.png")))
 test_dataset = BEVImageDataset(input_filepaths)
 im, sample_token = test_dataset[1]
im = im.numpy()
plt.figure(figsize=(16,8))
plt.imshow(im.transpose(1,2,0)[...,:3])
 plt.title(sample_token)
 plt.show()
visualize_lidar_of_sample(sample_token )<set_options>
FOLDS = 5
SEED = 1111
TRAINING = True
USE_FINETUNE = True
FILE_NAME = 'notebookec7152784a'
print('FOLDS : {}
SEED : {}
TRAINING : {}
USE_FINETUNE : {}
FILE_NAME : {}'.format(FOLDS, SEED, TRAINING,
USE_FINETUNE, FILE_NAME))
class UNet(nn.Module):
def __init__(
 self,
in_channels=1,
n_classes=2,
depth=5,
wf=6,
padding=False,
batch_norm=False,
up_mode='upconv', 
):

 super(UNet, self ).__init__()
assert up_mode in('upconv', 'upsample')
 self.padding = padding
self.depth = depth
prev_channels = in_channels
self.down_path = nn.ModuleList()
for i in range(depth):
self.down_path.append(
 UNetConvBlock(prev_channels, 2 **(wf + i), padding, batch_norm)
)
 prev_channels = 2 **(wf + i)
 self.up_path = nn.ModuleList()
for i in reversed(range(depth - 1)) :
self.up_path.append(
 UNetUpBlock(prev_channels, 2 **(wf + i), up_mode, padding, batch_norm)
)
 prev_channels = 2 **(wf + i)
 self.last = nn.Conv2d(prev_channels, n_classes, kernel_size=1)
 def forward(self, x):
blocks = []
for i, down in enumerate(self.down_path):
x = down(x)
 if i != len(self.down_path)- 1:
blocks.append(x)
 x = F.max_pool2d(x, 2)
 for i, up in enumerate(self.up_path):
x = up(x, blocks[-i - 1])
 return self.last(x)
 class UNetConvBlock(nn.Module):
def __init__(self, in_size, out_size, padding, batch_norm):
super(UNetConvBlock, self ).__init__()
block = []
block.append(nn.Conv2d(in_size, out_size, kernel_size=3, padding=int(padding)))
 block.append(nn.ReLU())
 if batch_norm:
block.append(nn.BatchNorm2d(out_size))
block.append(nn.Conv2d(out_size, out_size, kernel_size=3, padding=int(padding)))
 block.append(nn.ReLU())
 if batch_norm:
block.append(nn.BatchNorm2d(out_size))
self.block = nn.Sequential(*block)
 def forward(self, x):
out = self.block(x)
 return out
class UNetUpBlock(nn.Module):
def __init__(self, in_size, out_size, up_mode, padding, batch_norm):
super(UNetUpBlock, self ).__init__()
if up_mode == 'upconv':
self.up = nn.ConvTranspose2d(in_size, out_size, kernel_size=2, stride=2)
 elif up_mode == 'upsample':
self.up = nn.Sequential(
 nn.Upsample(mode='bilinear', scale_factor=2),
nn.Conv2d(in_size, out_size, kernel_size=1), 
)
 self.conv_block = UNetConvBlock(in_size, out_size, padding, batch_norm)
 def center_crop(self, layer, target_size):
_, _, layer_height, layer_width = layer.size()
diff_y =(layer_height - target_size[0])// 2
diff_x =(layer_width - target_size[1])// 2
return layer[
:, :, diff_y :(diff_y + target_size[0]), diff_x :(diff_x + target_size[1])
 ]
def forward(self, x, bridge):
up = self.up(x)
 crop1 = self.center_crop(bridge, up.shape[2:])
 out = torch.cat([up, crop1], 1)
 out = self.conv_block(out)
 return out<choose_model_class>
def check_random(SEED=SEED):
os.environ['TF_DETERMINISTIC_OPS'] = '1'
os.environ['PYTHONHASHSEED']=str(SEED)
 random.seed(SEED)
 np.random.seed(SEED)
 tf.random.set_seed(SEED)
 check_random()
def get_unet_model(in_channels=3, num_output_classes=2):
model = UNet(in_channels=in_channels, n_classes=num_output_classes, wf=5, depth=4, padding=True, up_mode='upsample')
 model = nn.DataParallel(model)
 return model 
<predict_on_test>
print('load train start')
 try:
train = pd.read_parquet('train.parquet')
 except:
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv', index_col='ts_id')
 train.to_parquet('train.parquet')
 print('load train end' )
def visualize_predictions(input_image, prediction, n_images=2, apply_softmax=True):

 prediction = prediction[:n_images]
input_image = input_image[:n_images]
prediction = prediction.detach().cpu().numpy()
if apply_softmax:
prediction = scipy.special.softmax(prediction, axis=1)
 class_one_preds = np.hstack(1-prediction[:,0])
 class_rgb = np.repeat(class_one_preds[..., None], 3, axis=2)
 class_rgb[...,2] = 0
input_im = np.hstack(input_image.cpu().numpy().transpose(0,2,3,1))
if input_im.shape[2] == 3:
input_im_grayscale = np.repeat(input_im.mean(axis=2)[..., None], 3, axis=2)
 overlayed_im =(input_im_grayscale*0.6 + class_rgb*0.7 ).clip(0,1)
 else:
input_map = input_im[...,3:]
overlayed_im =(input_map*0.6 + class_rgb*0.7 ).clip(0,1)
 thresholded_pred = np.repeat(class_one_preds[..., None] > 0.5, 3, axis=2)
 fig = plt.figure(figsize=(12,26))
plot_im = np.vstack([class_rgb, input_im[...,:3], overlayed_im, thresholded_pred] ).clip(0,1 ).astype(np.float32)
 plt.imshow(plot_im)
 plt.axis("off")
 plt.show()<categorify>
train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() ,inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if "feature" in c]
f_mean = np.mean(train[features[1:]].values,axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 class_weights = torch.from_numpy(np.array([0.2] + [1.0]*len(classes), dtype=np.float32))
class_weights = class_weights.to(device )<load_pretrained>
def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
inp_2 = tf.keras.backend.cos(inp)
 x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x_2 = tf.keras.layers.BatchNormalization()(inp_2)
 x_2 = tf.keras.layers.Dropout(dropout_rates[0] )(x_2)
 for i in range(len(hidden_units)) :
x_2 = tf.keras.layers.Dense(hidden_units[i] )(x_2)
 x_2 = tf.keras.layers.BatchNormalization()(x_2)
 x_2 = tf.keras.layers.Activation(tf.keras.activations.swish )(x_2)
 x_2 = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x_2)
 x = tf.keras.layers.Concatenate(axis=-1 )([x, x_2])
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
batch_size = 8
epochs = 15
model = get_unet_model(num_output_classes=len(classes)+1)
 state = torch.load('.. /input/reference-model/artifacts/unet_checkpoint_epoch_15.pth')
 model.load_state_dict(state)
 model = model.to(device)
 model.eval() ; 
<categorify>
batch_size = 5000
hidden_units = [150, 150, 150]
dropout_rates = [0.20, 0.20, 0.20, 0.20]
label_smoothing = 1e-2
learning_rate = 1e-3
clf = create_mlp(
 len(features), 5, hidden_units, dropout_rates, label_smoothing, learning_rate
)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
def open_preds(predictions_non_class0):
predictions_opened = np.zeros(( predictions_non_class0.shape), dtype=np.uint8)
 for i, p in enumerate(tqdm(predictions_non_class0)) :
thresholded_p =(p > background_threshold ).astype(np.uint8)
 predictions_opened[i] = cv2.morphologyEx(thresholded_p, cv2.MORPH_OPEN, kernel)
 return predictions_opened<define_variables>
gc.collect()
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size, shuffle=True, num_workers=os.cpu_count() *2)
 progress_bar = tqdm_notebook(test_loader)
 sample_tokens = []
all_losses = []
detection_boxes = []
detection_scores = []
detection_classes = []
background_threshold = 200
with torch.no_grad() :
model.eval()
for ii,(X, batch_sample_tokens)in enumerate(progress_bar):
sample_tokens.extend(batch_sample_tokens)
 X = X.to(device)
 prediction = model(X)
 prediction = F.softmax(prediction, dim=1)
 prediction_cpu = prediction.cpu().numpy()
predictions = np.round(prediction_cpu*255 ).astype(np.uint8)
 predictions_non_class0 = 255 - predictions[:,0]
predictions_opened = np.zeros(( predictions_non_class0.shape), dtype=np.uint8)
 for i, p in enumerate(predictions_non_class0):
thresholded_p =(p > background_threshold ).astype(np.uint8)
 predictions_opened[i] = cv2.morphologyEx(thresholded_p, cv2.MORPH_OPEN, kernel)
 sample_boxes,sample_detection_scores,sample_detection_classes = calc_detection_box(predictions_opened[i],
predictions[i])
 detection_boxes.append(np.array(sample_boxes))
detection_scores.append(sample_detection_scores)
 detection_classes.append(sample_detection_classes)
<prepare_x_and_y>
check_random()
clf.fit(X_train, y_train, epochs=200, batch_size=5000 )
def create_transformation_matrix_to_voxel_space(shape, voxel_size, offset):

 shape, voxel_size, offset = np.array(shape), np.array(voxel_size), np.array(offset)
 tm = np.eye(4, dtype=np.float32)
 translation = shape/2 + offset/voxel_size
tm = tm * np.array(np.hstack(( 1/voxel_size, [1])))
 tm[:3, 3] = np.transpose(translation)
 return tm
def transform_points(points, transf_matrix):

 if points.shape[0] not in [3,4]:
raise Exception("Points input should be(3,N)or(4,N)shape, received {}".format(points.shape))
return transf_matrix.dot(np.vstack(( points[:3, :], np.ones(points.shape[1])))) [:3, :]
def car_to_voxel_coords(points, shape, voxel_size, z_offset=0):
if len(shape)!= 3:
raise Exception("Voxel volume shape should be 3 dimensions(x,y,z)")
 if len(points.shape)!= 2 or points.shape[0] not in [3, 4]:
raise Exception("Input points should be(3,N)or(4,N)in shape, found {}".format(points.shape))
tm = create_transformation_matrix_to_voxel_space(shape, voxel_size,(0, 0, z_offset))
p = transform_points(points, tm)
 return p
def create_voxel_pointcloud(points, shape, voxel_size=(0.5,0.5,1), z_offset=0):
points_voxel_coords = car_to_voxel_coords(points.copy() , shape, voxel_size, z_offset)
 points_voxel_coords = points_voxel_coords[:3].transpose(1,0)
 points_voxel_coords = np.int0(points_voxel_coords)
 bev = np.zeros(shape, dtype=np.float32)
 bev_shape = np.array(shape)
 within_bounds =(np.all(points_voxel_coords >= 0, axis=1)* np.all(points_voxel_coords < bev_shape, axis=1))
points_voxel_coords = points_voxel_coords[within_bounds]
coord, count = np.unique(points_voxel_coords, axis=0, return_counts=True)
 bev[coord[:,1], coord[:,0], coord[:,2]] = count
return bev
def normalize_voxel_intensities(bev, max_intensity=16):
return(bev/max_intensity ).clip(0,1)
<prepare_output>
models = []
models.append(clf)
 th = 0.5000
print(th )
<save_to_csv><EOS> 
print('predict')
 f = np.median
models = models[-3:]
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )