Kaggle competition: VinBigData Chest X-ray Abnormalities Detection
<SOS> metric: OpenImagesObjectDetectionAP Kaggle data source: vinbigdata-chest-x-ray-abnormalities-detection<define_variables>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
dataset_path = ".. /input/global-wheat-detection/"
trainImg_path = dataset_path+"train/" 
<train_model>
pred_2class = pd.read_csv(".. /input/vinbigdata-2class-prediction/2-cls test pred.csv")
 pred_2class
f = open('wheat.yaml', 'w')
 f.write('train:./training_data
')
 f.write('val:./training_data
')
 f.write('test:./testing_data
')
 f.write('
')
 f.write('nc: 1
')
 f.write('
')
 f.write("names: ['wheat']
")
 f.close()<init_hyperparams>
low_threshold = 0.005
high_threshold = 0.95
<set_options>
commits_df = pd.DataFrame(columns = ['n_commit', 'low', 'high', 'LB_score'] )
!rm submission.csv<save_to_csv>
n=0
commits_df.loc[n, 'n_commit'] = 0
commits_df.loc[n, 'low'] = 0.001
commits_df.loc[n, 'high'] = 0.87
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.226
','')
<load_from_csv>
n=1
commits_df.loc[n, 'n_commit'] = n
commits_df.loc[n, 'low'] = 0.001
commits_df.loc[n, 'high'] = 0.90
commits_df.loc[n, 'LB_score'] = 0.244
commits_df.loc[n, 'LB_private'] = 0.226
!cat./submission.csv<load_from_csv>
n=2
commits_df.loc[n, 'n_commit'] = n
commits_df.loc[n, 'low'] = 0.001
commits_df.loc[n, 'high'] = 0.94
commits_df.loc[n, 'LB_score'] = 0.241
commits_df.loc[n, 'LB_private'] = 0.228
df = pd.read_csv('submission.csv')
 print(df )<define_variables>
n=3
commits_df.loc[n, 'n_commit'] = n
commits_df.loc[n, 'low'] = 0.0
commits_df.loc[n, 'high'] = 0.91
commits_df.loc[n, 'LB_score'] = 0.243
commits_df.loc[n, 'LB_private'] = 0.226
train_dir ='/kaggle/input/global-wheat-detection/train/'
test_dir = '/kaggle/input/global-wheat-detection/test/'
train = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 weights = '.. /input/fasterrcnn-resnet50-fpn/fasterrcnn_resnet50_fpn(1 ).pth'<load_from_csv>
n=4
commits_df.loc[n, 'n_commit'] = n
commits_df.loc[n, 'low'] = 0.002
commits_df.loc[n, 'high'] = 0.9
commits_df.loc[n, 'LB_score'] = 0.244
commits_df.loc[n, 'LB_private'] = 0.226
sample = pd.read_csv('.. /input/global-wheat-detection/sample_submission.csv')
 sample.shape<feature_engineering>
n=5
commits_df.loc[n, 'n_commit'] = n
commits_df.loc[n, 'low'] = 0.005
commits_df.loc[n, 'high'] = 0.95
commits_df.loc[n, 'LB_score'] = 0.241
commits_df.loc[n, 'LB_private'] = 0.228
class Wheatdatasets(Dataset):
def __init__(self,dataframe,image_dir,transforms = None):
super().__init__()
self.image_id = dataframe['image_id'].unique()
self.df = dataframe
self.img_dir = image_dir
self.transforms = transforms
def __getitem__(self,index:int):
image_id = self.image_id[index]
record = self.df[self.df['image_id']==image_id]
image = cv2.imread(self.img_dir+image_id+'.jpg',cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB ).astype(np.float32)
 image = image/255.0
if self.transforms:
sample ={
'image':image
}
sample =self.transforms(**sample)
 image = sample['image']
return image,image_id
def __len__(self)-> int:
return self.image_id.shape[0]<categorify>
n=6
commits_df.loc[n, 'n_commit'] = n+1
commits_df.loc[n, 'low'] = 0.001
commits_df.loc[n, 'high'] = 0.9
commits_df.loc[n, 'LB_score'] = 0.244
commits_df.loc[n, 'LB_private'] = 0.226
def get_transform() :
return al.Compose([
ToTensorV2(p=1.0)
 ] )<predict_on_test>
n=7
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.001
commits_df.loc[n, 'high'] = 0.88
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.226
model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained = False,pretrained_backbone=False )<categorify>
n=8
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.001
commits_df.loc[n, 'high'] = 0.86
commits_df.loc[n, 'LB_score'] = 0.245
commits_df.loc[n, 'LB_private'] = 0.226
x = model.to(device )<create_dataframe>
n=9
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.001
commits_df.loc[n, 'high'] = 0.875
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.225
def collate_fn(batch):
return tuple(zip(*batch))
test_dataset = Wheatdatasets(sample,test_dir,get_transform())
 test_dataloader = DataLoader(
 test_dataset,
batch_size=4,
shuffle =False,
num_workers =4,drop_last = False,
collate_fn = collate_fn)
<define_variables>
n=10
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.0
commits_df.loc[n, 'high'] = 0.875
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.225
def format_string(boxes,score):
pred_strings = []
for j in zip(score,boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
<categorify>
n=11
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.002
commits_df.loc[n, 'high'] = 0.885
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.226
detection_threshold = 0.5
results = []
for images, image_ids in test_dataloader:
images = list(image.to(device)for image in images)
 outputs = model(images)
 for i, image in enumerate(images):
boxes = outputs[i]['boxes'].data.cpu().numpy()
scores = outputs[i]['scores'].data.cpu().numpy()
boxes = boxes[scores >= detection_threshold].astype(np.int32)
 scores = scores[scores >= detection_threshold]
image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_string(boxes, scores)
 }
results.append(result )<prepare_output>
n=12
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.003
commits_df.loc[n, 'high'] = 0.88
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.226
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.head()<data_type_conversions>
n=13
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.005
commits_df.loc[n, 'high'] = 0.88
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.226
<save_to_csv>
n=14
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.01
commits_df.loc[n, 'high'] = 0.88
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.226
test_df.to_csv('submission.csv', index=False )<define_variables>
n=15
commits_df.loc[n, 'n_commit'] = n+3
commits_df.loc[n, 'low'] = 0.1
commits_df.loc[n, 'high'] = 0.88
commits_df.loc[n, 'LB_score'] = 0.246
commits_df.loc[n, 'LB_private'] = 0.226
img_id = list(test_df['image_id'])
 print(img_id )<create_dataframe>
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df['LB_private'] = pd.to_numeric(commits_df['LB_private'])
 commits_df = commits_df.sort_values(by=['LB_score'], ascending = False ).reset_index(drop=True)
 commits_df['max'] = 0
commits_df.loc[15, 'max'] = 1
commits_df.loc[0, 'max'] = 2
commits_df['max_private'] = 0
commits_df.loc[0, 'max_private'] = 1
commits_df.loc[15, 'max_private'] = 2
for i in img_id:
q = df.loc[df['image_id'] == i]
if q.empty:
print(i)
 p = test_df.loc[test_df['image_id'] == i]['PredictionString'].item()
df2 = pd.DataFrame([[i, p]], columns=['image_id','PredictionString'])
 print(df2)
 df = df.append(df2 )<save_to_csv>
commits_df.sort_values(by=['LB_score'], ascending = True )
df.to_csv('submission.csv', index=False )<define_variables>
commits_df.sort_values(by=['LB_private'], ascending = True )
<compute_train_metric><EOS> 
NORMAL = "14 1 0 0 1 1"
pred_det_df = pd.read_csv(".. /input/vinbigdatastack/submission_postprocessed.csv")
 n_normal_before = len(pred_det_df.query("PredictionString == @NORMAL"))
merged_df = pd.merge(pred_det_df, pred_2class, on="image_id", how="left")
 if "target" in merged_df.columns:
merged_df["class0"] = 1 - merged_df["target"]
c0, c1, c2 = 0, 0, 0
for i in range(len(merged_df)) :
p0 = merged_df.loc[i, "class0"]
if p0 < low_threshold:
c0 += 1
elif low_threshold <= p0 and p0 < high_threshold:
merged_df.loc[i, "PredictionString"] += f" 14 {p0} 0 0 1 1"
c1 += 1
else:
merged_df.loc[i, "PredictionString"] = NORMAL
c2 += 1
n_normal_after = len(merged_df.query("PredictionString == @NORMAL"))
print(
 f"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}" 
)
 print(f"Keep {c0} Add {c1} Replace {c2}")
 submission_filepath = str("submission.csv")
 submission_df = merged_df[["image_id", "PredictionString"]]
submission_df.to_csv(submission_filepath, index=False)
 print(f"Saved to {submission_filepath}" )