Kaggle competition: VinBigData Chest X-ray Abnormalities Detection
<SOS> metric: OpenImagesObjectDetectionAP Kaggle data source: vinbigdata-chest-x-ray-abnormalities-detection<train_model>
!pip install -U ensemble-boxes
<train_model>
import pandas as pd
import numpy as np
from ensemble_boxes import *
from glob import glob
import copy
from tqdm import tqdm
import shutil
model = create_model()
model.summary()
model.compile(optimizer=keras.optimizers.SGD() ,
loss='categorical_crossentropy',
metrics=['accuracy'])
<train_model>
height_dict = pd.read_csv('.. /input/vinbigdata-original-image-dataset/vinbigdata/test.csv' ).to_dict('records')
 fnl_dict ={}
for ix,i in enumerate(height_dict):
fnl_dict[i['image_id']] = [i['width'],i['height'],i['width'],i['height']]
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<choose_model_class>
subs = [
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_1.csv'),
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_2.csv'),
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_3.csv'),
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_4.csv'),
pd.read_csv('.. /input/yolo-vbd-lots-of-decimals/Fold_5.csv')
 ]
pred_2cls = pd.read_csv('.. /input/vinbigdata-2-class-classifier-complete-pipeline/results/tmp_debug/test_pred.csv' )
def create_my_model() :
input_layer = Input(shape=(28, 28, 1))
x = Conv2D(filters=32, kernel_size=(5, 5), padding='Same', activation='relu' )(input_layer)
 x = Conv2D(filters=32, kernel_size=(5, 5), padding='Same', activation='relu' )(x)
 x = BatchNormalization(momentum=.15 )(x)
 x = MaxPool2D(pool_size=(2,2))(x)
 x = Dropout(0.25 )(x)
 x = Conv2D(filters=64, kernel_size=(3, 3), padding='Same', activation='relu' )(x)
 x = Conv2D(filters=64, kernel_size=(3, 3), padding='Same', activation='relu' )(x)
 x = BatchNormalization(momentum=.15 )(x)
 x = MaxPool2D(pool_size=(2,2))(x)
 x = Dropout(0.25 )(x)
 x = Conv2D(filters=32, kernel_size=(5, 5), padding='Same', activation='relu' )(x)
 x = Conv2D(filters=32, kernel_size=(5, 5), padding='Same', activation='relu' )(x)
 x = BatchNormalization(momentum=.15 )(x)
 x = MaxPool2D(pool_size=(2,2))(x)
 x = Flatten()(x)
 x = Dense(256, activation="relu" )(x)
 x = Dropout(0.4 )(x)
 output = Dense(10, activation='softmax' )(x)
 model = Model(inputs=[input_layer], outputs=[output])
 return model<choose_model_class>
def submission_decoder(df:pd.DataFrame)-> np.ndarray:
info = df.values
df_lst = []
for i in info:
pre_lst = i[1].split(' ')
 for j in range(0,len(pre_lst),6):
df_lst.append([i[0],int(pre_lst[j]),float(pre_lst[j+1]),int(pre_lst[j+2]),int(pre_lst[j+3]),\
int(pre_lst[j+4]),int(pre_lst[j+5]),fnl_dict.get(i[0])[0],fnl_dict.get(i[0])[1]])
 return pd.DataFrame(df_lst,columns = ['image_id','class_id','score','x_min','y_min','x_max','y_max','width','height'] )
my_model.compile(optimizer=keras.optimizers.Adam() ,
loss='categorical_crossentropy',
metrics=['accuracy'] )<train_model>
subs = [submission_decoder(subs[i])for i in range(len(subs)) ]
my_model.fit(datagen.flow(X_train, y_train, batch_size=64),
epochs=30,
verbose=1, validation_data =(X_valid, y_valid),
callbacks=[ReduceLROnPlateau(monitor='val_accuracy',
factor=0.5,
patieecne=3,
verbose=1,
min_lr=0.00001),] )<save_to_csv>
boxes_dict = {}
scores_dict = {}
labels_dict = {}
whwh_dict = {}
for i in tqdm(subs[0].image_id.unique()):
if not i in boxes_dict.keys() :
boxes_dict[i] = []
scores_dict[i] = []
labels_dict[i] = []
whwh_dict[i] = []
size_ratio = fnl_dict.get(i)
 whwh_dict[i].append(size_ratio)
 tmp_df = [subs[x][subs[x]['image_id']==i] for x in range(len(subs)) ]
for x in range(len(tmp_df)) :
boxes_dict[i].append(((tmp_df[x][['x_min','y_min','x_max','y_max']].values)/size_ratio ).tolist())
 scores_dict[i].append(tmp_df[x]['score'].values.tolist())
 labels_dict[i].append(tmp_df[x]['class_id'].values.tolist() )
y_result = my_model.predict(X_test)
 y_result = np.argmax(y_result, axis=1)
 sample_df['label'] = y_result
sample_df.to_csv('submission.csv', index=False )<choose_model_class>
weights = [1]*5
weights1 = [3,2,4,5]
iou_thr = 0.5
skip_box_thr = 0.0001
sigma = 0.1
fnl = {}
for i in tqdm(boxes_dict.keys()):
boxes, scores, labels = weighted_boxes_fusion(boxes_dict[i], scores_dict[i], labels_dict[i],\
weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 if not i in fnl.keys() :
fnl[i] = {'boxes':[],'scores':[],'labels':[]}
fnl[i]['boxes'] = boxes*whwh_dict[i]
fnl[i]['scores'] = scores
fnl[i]['labels'] = labels
optimizers = {
'sgd': 'SGD',
'sgd_mom': keras.optimizers.SGD(momentum=0.5),
'rmsprop': 'RMSprop',
'adam': 'Adam',
} 
<train_model>
pd_form = []
for i in fnl.keys() :
b = fnl[i]
for j in range(len(b['boxes'])) :
pd_form.append([i,int(b['labels'][j]),round(b['scores'][j],2),\
int(b['boxes'][j][0]),int(b['boxes'][j][1]),\
int(b['boxes'][j][2]),int(b['boxes'][j][3])])
 final_df = pd.DataFrame(pd_form,columns = ['image_id','class_id','score','x_min','y_min','x_max','y_max'])
 final_df = final_df.drop_duplicates(keep = 'first' )
historys = {}
models = {}
num_epochs = 1
for n, optimizer in enumerate(optimizers):
models[optimizer] = create_my_model()
models[optimizer].compile(optimizer=optimizers[optimizer],
loss='categorical_crossentropy',
metrics=['accuracy'])
 start = time.time()
h = models[optimizer].fit(X_train, y_train,
batch_size=1024,
epochs=num_epochs,
verbose=0,
validation_data=(X_valid, y_valid),
callbacks=[ReduceLROnPlateau(monitor='val_accuracy',
factor=0.5,
patieecne=3,
verbose=1,
min_lr=0.00001),
CSVLogger("{0}.log".format(optimizer)) ] 
)
 historys[optimizer] = h.history
print("{0} Optimizer: ".format(optimizer))
print("accuracy: {0}, val_accuracy {1}, time {2} minutes: ".format(\
historys[optimizer]['accuracy'][-1],
historys[optimizer]['val_accuracy'][-1], 
(time.time() - start)/ 60))<import_modules>
def submission_encoder(df:pd.DataFrame)-> np.ndarray:
dct = {}
for i in tqdm(df['image_id'].unique()):
if not i in dct.keys() :
dct[i] = []
tmp = df[df['image_id'] == i].values
for j in tmp:
dct[i].append(int(j[1]))
dct[i].append(float(j[2]))
dct[i].append(int(j[3]))
dct[i].append(int(j[4]))
dct[i].append(int(j[5]))
dct[i].append(int(j[6]))
dct[i] = map(str,dct[i])
 dct[i] = ' '.join(dct[i])
 dct = [[k, v] for k, v in dct.items() ]
return pd.DataFrame(dct,columns = ['image_id','PredictionString'] ).reset_index(drop = True)
 df = submission_encoder(final_df)
 df.to_csv('Fold5Yolo.csv', index=False )
<install_modules><EOS> 
NORMAL = "14 1 0 0 1 1"
low_threshold = 0.00
high_threshold = 0.99
pred_det_df = df
n_normal_before = len(pred_det_df.query("PredictionString == @NORMAL"))
merged_df = pd.merge(pred_det_df, pred_2cls, on="image_id", how="left")
 if "target" in merged_df.columns:
merged_df["class0"] = 1 - merged_df["target"]
c0, c1, c2 = 0, 0, 0
for i in range(len(merged_df)) :
p0 = merged_df.loc[i, "class0"]
if p0 < low_threshold:
c0 += 1
elif low_threshold <= p0 and p0 < high_threshold:
if ' 14 ' not in merged_df.loc[i, "PredictionString"]:
merged_df.loc[i, "PredictionString"] += f" 14 {p0} 0 0 1 1"
c1 += 1
else:
merged_df.loc[i, "PredictionString"] = NORMAL
c2 += 1
n_normal_after = len(merged_df.query("PredictionString == @NORMAL"))
print(
 f"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}" 
)
 print(f"Keep {c0} Add {c1} Replace {c2}")
 submission_filepath = str("submission.csv")
 submission_df = merged_df[["image_id", "PredictionString"]]
submission_df.to_csv(submission_filepath, index=False)
 print(f"Saved to {submission_filepath}" )