Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<init_hyperparams>
!pip install pytorch_tabular
!pip install torch_optimizer
IMG_HEIGHT = 736
IMG_WIDTH = 1184
features = VGG16(weights='imagenet',
include_top=False,
input_shape=(IMG_HEIGHT, IMG_WIDTH, 3))
for layer in features.layers[:-5]:
layer.trainable = False
feature_tensor = features.layers[-1].output
print(feature_tensor.shape )<define_variables>
import numpy as np
import pandas as pd
import time
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import StandardScaler
from pytorch_tabular import TabularModel
from pytorch_tabular.models import CategoryEmbeddingModelConfig, NodeConfig, TabNetModelConfig
from pytorch_tabular.config import DataConfig, OptimizerConfig, TrainerConfig, ExperimentConfig
from pytorch_tabular.categorical_encoders import CategoricalEmbeddingTransformer
from torch_optimizer import QHAdam
import category_encoders as ce
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor
FEATURE_SHAPE =(feature_tensor.shape[1],
feature_tensor.shape[2])
 print(FEATURE_SHAPE)
 GRID_STEP_H = IMG_HEIGHT / FEATURE_SHAPE[0]
GRID_STEP_W = IMG_WIDTH / FEATURE_SHAPE[1]
ANCHOR_WIDTH = 150.
 ANCHOR_HEIGHT = 150.
 ANCHOR_CENTERS = np.mgrid[GRID_STEP_H/2:IMG_HEIGHT:GRID_STEP_H,
GRID_STEP_W/2:IMG_WIDTH:GRID_STEP_W]
ANCHOR_CENTERS.shape<define_variables>
df_train = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 display(df_train.head())
 df_test = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv')
 display(df_test.head())
 features = ['cont1', 'cont2', 'cont3', 'cont4', 'cont5',
'cont6', 'cont7', 'cont8', 'cont9', 'cont10',
'cont11', 'cont12', 'cont13', 'cont14']
num_classes = counts.shape[0]
num_classes<import_modules>
enc = KBinsDiscretizer(n_bins=10, encode='ordinal', strategy="quantile")
 enc.fit(df_train[features])
 binned_df_train = enc.transform(df_train[features])
 binned_df_test = enc.transform(df_test[features])
 for i, feature in enumerate(features):
df_train[f"{feature}_binned"] = binned_df_train[:,i]
df_test[f"{feature}_binned"] = binned_df_test[:,i]
from scipy.special import softmax<normalization>
def get_configs(train):
epochs = 25
batch_size = 512
steps_per_epoch = int(( len(train)//batch_size)*0.9)
 data_config = DataConfig(
 target=['target'],
continuous_cols=['cont1', 'cont2', 'cont3', 'cont4', 'cont5', 'cont6', 'cont7',
'cont8', 'cont9', 'cont10', 'cont11', 'cont12', 'cont13', 'cont14'],
categorical_cols=['cont1_binned', 'cont2_binned', 'cont3_binned',
'cont4_binned', 'cont5_binned', 'cont6_binned', 'cont7_binned',
'cont8_binned', 'cont9_binned', 'cont10_binned', 'cont11_binned',
'cont12_binned', 'cont13_binned', 'cont14_binned'],
continuous_feature_transform="quantile_normal" 
)
 trainer_config = TrainerConfig(
 auto_lr_find=False,
batch_size=batch_size,
max_epochs=epochs,
gpus=1, 
)
 optimizer_config = OptimizerConfig(lr_scheduler="OneCycleLR", lr_scheduler_params={"max_lr":0.005, "epochs": epochs, "steps_per_epoch":steps_per_epoch})
 model_config = NodeConfig(
 task="regression",
num_layers=2,
num_trees=1024,
depth=6,
embed_categorical=True,
learning_rate = 1e-3,
target_range=[(df_train[col].min() ,df_train[col].max())for col in ['target']] 
)
 return data_config, trainer_config, optimizer_config, model_config
def iou(rect, x_scale, y_scale, anchor_x, anchor_y,
anchor_w=ANCHOR_WIDTH, anchor_h=ANCHOR_HEIGHT):
rect_x1 =(rect['x'] - rect['width'] / 2)* x_scale
rect_x2 =(rect['x'] + rect['width'] / 2)* x_scale
rect_y1 =(rect['y'] - rect['height'] / 2)* y_scale
rect_y2 =(rect['y'] + rect['height'] / 2)* y_scale
anch_x1, anch_x2 = anchor_x - anchor_w / 2, anchor_x + anchor_w / 2
anch_y1, anch_y2 = anchor_y - anchor_h / 2, anchor_y + anchor_h / 2
dx =(min(rect_x2, anch_x2)- max(rect_x1, anch_x1))
dy =(min(rect_y2, anch_y2)- max(rect_y1, anch_y1))
intersection = dx * dy if(dx > 0 and dy > 0)else 0.
 anch_square =(anch_x2 - anch_x1)*(anch_y2 - anch_y1)
 rect_square =(rect_x2 - rect_x1)*(rect_y2 - rect_y1)
 union = anch_square + rect_square - intersection
return intersection / union
def encode_anchors(annotation, img_shape, iou_thr=0.5):
encoded = np.zeros(shape=(FEATURE_SHAPE[0],
FEATURE_SHAPE[1], 11), dtype=np.float32)
 x_scale = 1.* IMG_WIDTH / img_shape[1]
y_scale = 1.* IMG_HEIGHT / img_shape[0]
for rect in annotation['annotations']:
scores = []
label = fish_classes == rect['class']
for row in range(FEATURE_SHAPE[0]):
for col in range(FEATURE_SHAPE[1]):
anchor_x = ANCHOR_CENTERS[1, row, col]
anchor_y = ANCHOR_CENTERS[0, row, col]
score = iou(rect, x_scale, y_scale, anchor_x, anchor_y)
 scores.append(( score, anchor_x, anchor_y, row, col))
scores = sorted(scores, reverse=True)
 if scores[0][0] < iou_thr:
scores = [scores[0]]
else:
scores = [e for e in scores if e[0] > iou_thr]
for score, anchor_x, anchor_y, row, col in scores:
dx =(anchor_x - rect['x'] * x_scale)/ ANCHOR_WIDTH
dy =(anchor_y - rect['y'] * y_scale)/ ANCHOR_HEIGHT
dw =(ANCHOR_WIDTH - rect['width'] * x_scale)/ ANCHOR_WIDTH
dh =(ANCHOR_HEIGHT - rect['height'] * y_scale)/ ANCHOR_HEIGHT
encoded[row, col] = np.array([*label, 1., dx, dy, dw, dh])
 return encoded
def _sigmoid(x):
return 1./(1.+ np.exp(-x))
def decode_prediction(prediction, conf_thr=0.1):
rectangles = []
for row in range(FEATURE_SHAPE[0]):
for col in range(FEATURE_SHAPE[1]):
label = np.empty(6)
 label[0], label[1], label[2], label[3], label[4], label[5], conf, dx, dy, dw, dh = prediction[row, col]
conf = _sigmoid(conf)
 label = softmax(label)
 if conf > conf_thr:
anchor_x = ANCHOR_CENTERS[1, row, col]
anchor_y = ANCHOR_CENTERS[0, row, col]
rectangles.append({'x': anchor_x - dx * ANCHOR_WIDTH,
'y': anchor_y - dy * ANCHOR_HEIGHT,
'width': ANCHOR_WIDTH - dw * ANCHOR_WIDTH,
'height': ANCHOR_HEIGHT - dh * ANCHOR_HEIGHT,
'conf': conf,
'label': np.argmax(_sigmoid(label)) ,
'labels': label })
 return rectangles
def non_max_suppression(rectangles, max_output_size, iou_threshold=0.5):
if not rectangles:
return rectangles
boxes = [[r['y'],
r['x'],
r['y'] + r['height'],
r['x'] + r['width']] for r in rectangles]
scores = [r['conf'] for r in rectangles]
indices = tf.image.non_max_suppression(np.array(boxes),
np.array(scores),
max_output_size,
iou_threshold)
 return [rectangles[i] for i in indices]<categorify>
rnd_seed_cv = 1234
rnd_seed_reg = 1234
kf = KFold(n_splits=5, random_state=rnd_seed_cv, shuffle=True)
 df_train.drop(columns='id', inplace=True)
 df_test.drop(columns='id', inplace=True)
 df_test['target'] = 0
annotation = boxes['shark'][3]
encoded = encode_anchors(annotation,
img_shape=(IMG_HEIGHT, IMG_WIDTH),
iou_thr=0.1)
 decoded = decode_prediction(encoded, conf_thr=0.7)
 decoded = sorted(decoded, key = lambda e: -e['conf'])
 plt.figure(figsize=(6, 6), dpi=240)
 plt.imshow(draw_boxes(annotation, decoded))
plt.title('{} {}x{}'.format(
 annotation['filename'], img.shape[0], img.shape[1])) ;<compute_train_metric>
def node(train, valid, df_test):
data_config, trainer_config, optimizer_config, model_config = get_configs(train)
 tabular_model = TabularModel(
 data_config=data_config,
model_config=model_config,
optimizer_config=optimizer_config,
trainer_config=trainer_config 
)
 tabular_model.fit(train=train, validation=valid, optimizer=QHAdam,
optimizer_params={"nus":(0.7, 1.0), "betas":(0.95, 0.998)})
 result = tabular_model.evaluate(valid)
 return np.sqrt(result[0]["test_mean_squared_error"]), tabular_model.predict(valid)["target_prediction"].values, tabular_model.predict(df_test)["target_prediction"].values, tabular_model
def lgbm(train, valid, df_test):
lgb_model = LGBMRegressor(n_estimators=10000,
learning_rate=0.005,
early_stopping_rounds=50,
feature_pre_filter = False,
num_leaves=102,
min_child_samples=20,
colsample_bytree = 0.4,
subsample = 1,
subsample_freq = 0,
lambda_l1 = 4.6,
lambda_l2 = 1.9,
random_state=42)
 lgb_model.fit(train.drop(columns='target'), train['target'], eval_set=(valid.drop(columns='target'),valid.loc[:,'target']))
lgb_preds = lgb_model.predict(valid.drop(columns='target'))
score = mean_squared_error(valid['target'].values, lgb_preds, squared=False)
 return score, lgb_model.predict(valid.drop(columns='target')) , lgb_model.predict(df_test.drop(columns='target'))
def cb(train, valid, df_test):
best_params = {
'grow_policy': 'Lossguide',
'boosting_type': 'Plain',
'depth': 20,
'l2_leaf_reg': 3.699746597668451,
'min_data_in_leaf': 4,
'random_strength': 4.9263987954247455,
'rsm': 1.0,
"n_estimators": 10000,
"learning_rate": 0.5,
"od_type": "Iter",
"od_wait": 50,
}
catboost_model = CatBoostRegressor(**best_params)
 catboost_model.fit(train.drop(columns='target'), train['target'], eval_set=(valid.drop(columns='target'),valid.loc[:,'target']))
cb_preds = catboost_model.predict(valid.drop(columns='target'))
score = mean_squared_error(valid['target'].values, cb_preds, squared=False)
 return score, catboost_model.predict(valid.drop(columns='target')) , catboost_model.predict(df_test.drop(columns='target'))
K = tf.keras.backend
def confidence_loss(y_true, y_pred):
conf_loss = K.binary_crossentropy(y_true[..., 6],
y_pred[..., 6],
from_logits=True)
 return conf_loss
def smooth_l1(y_true, y_pred):
abs_loss = K.abs(y_true[..., -4:] - y_pred[..., -4:])
 square_loss = 0.5 * K.square(y_true[..., -4:] - y_pred[..., -4:])
 mask = K.cast(K.greater(abs_loss, 1.) , 'float32')
 total_loss =(abs_loss - 0.5)* mask + 0.5 * square_loss *(1.- mask)
 return K.sum(total_loss, axis=-1)
 def classification_loss(y_tr, y_pr, alpha=0.25, gamma=2.0):

 y_true = y_tr[..., :6]
y_pred = y_pr[..., :6]
cross_entropy = tf.nn.sigmoid_cross_entropy_with_logits(
 labels=y_true, logits=y_pred 
)
 probs = tf.nn.softmax(y_pred)
 alpha = tf.where(tf.equal(y_true, 1.0), alpha,(1.0 - alpha))
pt = tf.where(tf.equal(y_true, 1.0), probs, 1 - probs)
 loss = alpha * tf.pow(1.0 - pt, gamma)* cross_entropy
return tf.reduce_sum(loss, axis=-1)
 def class_loss(y_tr, y_pr):
y_true = y_tr[..., :6]
y_pred = y_pr[..., :6]
cross_entropy = K.categorical_crossentropy(y_true[..., :6],
y_pred[..., :6],
from_logits=True)
 return cross_entropy
def total_loss(y_true, y_pred, neg_pos_ratio=3):
batch_size = K.shape(y_true)[0]
y_true = K.reshape(y_true,(batch_size, -1, 11))
y_pred = K.reshape(y_pred,(batch_size, -1, 11))
cls_loss = classification_loss(y_true, y_pred)
 conf_loss = confidence_loss(y_true, y_pred)
 loc_loss = smooth_l1(y_true, y_pred)
 pos_conf_loss = K.sum(conf_loss * y_true[..., 6], axis=-1)
 pos_class_loss = K.sum(cls_loss * y_true[..., 6], axis=-1)
 pos_loc_loss = K.sum(loc_loss * y_true[..., 6], axis=-1)
 anchors = K.shape(y_true)[1]
num_pos = K.sum(y_true[..., 6], axis=-1)
 num_pos_avg = K.mean(num_pos)
 num_neg = K.min([neg_pos_ratio *(num_pos_avg)+ 1., K.cast(anchors, 'float32')])
 neg_conf_loss, _ = tf.nn.top_k(conf_loss *(1.- y_true[..., 6]),
k=K.cast(num_neg, 'int32'))
neg_conf_loss = K.sum(neg_conf_loss, axis=-1)
 total_conf_loss =(neg_conf_loss + pos_conf_loss)/(num_neg + num_pos + 1e-32)
 cls_loss = pos_class_loss /(num_pos + 1e-32)
 loc_loss = pos_loc_loss /(num_pos + 1e-32)
 return total_conf_loss + 0.5 * loc_loss + cls_loss 
<prepare_x_and_y>
CV_node = []
CV_lgb = []
CV_cb = []
preds_train_node = []
preds_train_lgb = []
preds_train_cb = []
preds_test_node = []
preds_test_lgb = []
preds_test_cb = []
cross_validated_preds = []
t1 = time.time()
for train_index, test_index in kf.split(df_train):
train = df_train.iloc[train_index]
valid = df_train.iloc[test_index]
cv_val = valid.copy()
node_score, node_train_pred, node_test_pred, tabular_model = node(train, valid, df_test)
 CV_node.append(node_score)
 cv_val['pred_node'] = node_train_pred
preds_train_node.append(node_train_pred)
 preds_test_node.append(node_test_pred)
 transformer = CategoricalEmbeddingTransformer(tabular_model)
 train_transform = transformer.fit_transform(train)
 val_transform = transformer.transform(valid)
 df_test_transform = transformer.transform(df_test)
 lgbm_score, lgbm_train_pred, lgbm_test_pred = lgbm(train_transform, val_transform, df_test_transform)
 CV_lgb.append(lgbm_score)
 cv_val['pred_lgb'] = lgbm_train_pred
preds_train_lgb.append(lgbm_train_pred)
 preds_test_lgb.append(lgbm_test_pred)
 cb_score, cb_train_pred, cb_test_pred = cb(train_transform, val_transform, df_test_transform)
 CV_cb.append(cb_score)
 cv_val['pred_cb'] = cb_train_pred
preds_train_cb.append(cb_train_pred)
 preds_test_cb.append(cb_test_pred)
 cross_validated_preds.append(cv_val)
 t2 = time.time()
print('Elapsed time [s]: ', t2-t1 )
def load_img(path, target_size=(IMG_WIDTH, IMG_HEIGHT)) :
img = cv2.imread(path, cv2.IMREAD_COLOR)[...,::-1]
img_shape = img.shape
img_resized = cv2.resize(img, target_size)
 return img_shape, tf.keras.applications.resnet_v2.preprocess_input(img_resized.astype(np.float32))
def data_generator(boxes, batch_size=32):
boxes = sum(boxes.values() , [])
 while True:
shuffle(boxes)
 for i in range(len(boxes)//batch_size):
X, y = [], []
for j in range(i*batch_size,(i+1)*batch_size):
img_shape, img = load_img(boxes[j]['filename'])
 y_ = encode_anchors(boxes[j], img_shape)
 y.append(y_)
 X.append(img)
 yield np.array(X), np.array(y )<choose_model_class>
print('CV performance [RMSE]: ', np.mean(CV_node, axis=0))
print('CV performance [RMSE]: ', np.mean(CV_lgb, axis=0))
print('CV performance [RMSE]: ', np.mean(CV_cb, axis=0))
output = tf.keras.layers.BatchNormalization()(feature_tensor)
 output = tf.keras.layers.Conv2D(11,
kernel_size=(1, 1),
activation='linear',
kernel_regularizer='l2' )(output)
 model = tf.keras.models.Model(inputs=features.inputs, outputs=output)
<define_search_space>
cross_val_pred_df = pd.concat(cross_validated_preds, sort=False)
 cross_val_pred_df.to_csv("cross_val_preds.csv")
 joblib.dump(preds_test_node, "preds_test_node.sav")
 joblib.dump(preds_test_lgb, "preds_test_lgb.sav")
 joblib.dump(preds_test_cb, "preds_test_cb.sav" )
lr=3e-3<choose_model_class>
avg_cb_pred = np.mean(preds_test_cb, axis=0)
 avg_lgb_pred = np.mean(preds_test_lgb, axis=0)
 avg_node_pred = np.mean(preds_test_node, axis=0)
 pred_test = np.average([avg_node_pred, avg_lgb_pred, avg_cb_pred], axis=0, weights=[-0.15447081, 1.1021915 , 0.06145868] )
def lr_exp_decay(epoch, lr):
k = 0.5
return lr * np.exp(-k*epoch)
 batch_size = 32
steps_per_epoch = sum(map(len, boxes.values()), 0)/ batch_size
gen = data_generator(boxes, batch_size=batch_size)
 checkpoint = tf.keras.callbacks.ModelCheckpoint(
 'fishdetector.hdf5',
monitor='loss',
verbose=47,
save_best_only=True,
save_weights_only=False,
mode='auto',
save_freq=10)
 adam = tf.keras.optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999)
 model.compile(optimizer=adam,
loss=total_loss,
metrics=[confidence_loss, classification_loss, smooth_l1])
<train_model>
df_sub = pd.read_csv('.. /input/tabular-playground-series-jan-2021/sample_submission.csv')
 df_sub.target = pred_test
df_sub.head()
model.fit(gen,
steps_per_epoch=steps_per_epoch,
epochs=2,
callbacks=[checkpoint, tf.keras.callbacks.LearningRateScheduler(lr_exp_decay, verbose=1)] )<predict_on_test>
df_sub.to_csv('submission.csv', index=False )