Kaggle competition: VinBigData Chest X-ray Abnormalities Detection
<SOS> metric: OpenImagesObjectDetectionAP Kaggle data source: vinbigdata-chest-x-ray-abnormalities-detection<prepare_x_and_y>
warnings.filterwarnings("ignore" )
pseudo_dataset = KannadaMNISTDataset(TEST_PATH, transform)
 pseudo_dataset.X = pseudo_dataset.X[good_ind]
pseudo_dataset.y = good_labels
pseudo_loader = torch.utils.data.DataLoader(pseudo_dataset, batch_size=BATCH_SIZE, num_workers=8, pin_memory=True)
 for lr in [1e-4, 1e-5, 1e-6]:
for g in optimizer.param_groups:
g['lr'] = lr
for epoch in tqdm(range(EPOCH, EPOCH + 100), leave=False):
net.train()
for batch_x, batch_y in tqdm(pseudo_loader, leave=False):
optimizer.zero_grad()
outputs = net(batch_x.cuda())
 loss = criterion(outputs, batch_y.cuda())
 loss.backward()
optimizer.step()
EPOCH += 1<save_to_csv>
train_data = pd.read_csv(path+'train.csv')
 samp_subm = pd.read_csv(path+'sample_submission.csv' )
test_transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.Resize(( 32, 32)) ,
transforms.ToTensor()
])
 test_dataset = KannadaMNISTDataset(TEST_PATH, test_transform)
 loader = torch.utils.data.DataLoader(test_dataset, batch_size=BATCH_SIZE)
 results = []
net.eval()
for batch_x, _ in tqdm(loader):
outputs = net(batch_x.cuda())
 results.extend(map(lambda x: x.item() , outputs.argmax(dim=1)))
 submission = pd.DataFrame({
'id': range(len(test_dataset)) ,
'label': results
})
 submission.to_csv(SUBMISSION_PATH, index=False )<set_options>
print('Number train samples:', len(train_data.index))
print('Number test samples:', len(samp_subm.index))
plt.style.use('ggplot')
 %matplotlib inline
,Flatten, Lambda, Reshape, LocallyConnected2D, BatchNormalization, MaxPool2D, Dropout 
<load_from_csv>
idnum = 2
image_id = train_data.loc[idnum, 'image_id']
data_file = dicom.dcmread(path+'train/'+image_id+'.dicom')
 img = data_file.pixel_array
dirname = '/kaggle/input/Kannada-MNIST'
output_dirname = '/kaggle/output/kaggle/working'
train_df = pd.read_csv(os.path.join(dirname, 'train.csv'))
test_df = pd.read_csv(os.path.join(dirname, 'test.csv'))
dig_df = pd.read_csv(os.path.join(dirname, 'Dig-MNIST.csv'))
sample_df = pd.read_csv(os.path.join(dirname, 'sample_submission.csv')) 
<categorify>
pred_2class = pd.read_csv(".. /input/vinbigdata-2class-prediction/2-cls test pred.csv")
 low_threshold = 0.001
high_threshold = 0.87
pred_2class
X = train_df.drop('label', axis=1 ).to_numpy()
y = train_df['label'].to_numpy()
X_dig = dig_df.drop('label', axis=1 ).to_numpy()
y_dig = dig_df['label'].to_numpy()
X_test = test_df.drop('id', axis=1 ).to_numpy()
X = X.reshape(-1, 28, 28, 1)
 y = to_categorical(y)
 X_dig = X_dig.reshape(-1, 28, 28, 1)
 y_dig = to_categorical(y_dig)
 X_test = X_test.reshape(-1, 28, 28, 1)
 X = X / 255
X_dig = X_dig / 255
X_test = X_test / 255
seed = 33
np.random.RandomState(seed)
 valid_size=0.15
X_train, X_valid, y_train, y_valid = train_test_split(X, y,\
test_size=valid_size, random_state=seed )<define_search_model>
NORMAL = "14 1 0 0 1 1"
pred_det_df = pd.read_csv(".. /input/vinbigdatastack/submission_postprocessed.csv")
 n_normal_before = len(pred_det_df.query("PredictionString == @NORMAL"))
merged_df = pd.merge(pred_det_df, pred_2class, on="image_id", how="left")
 if "target" in merged_df.columns:
merged_df["class0"] = 1 - merged_df["target"]
c0, c1, c2 = 0, 0, 0
for i in range(len(merged_df)) :
p0 = merged_df.loc[i, "class0"]
if p0 < low_threshold:
c0 += 1
elif low_threshold <= p0 and p0 < high_threshold:
merged_df.loc[i, "PredictionString"] += f" 14 {p0} 0 0 1 1"
c1 += 1
else:
merged_df.loc[i, "PredictionString"] = NORMAL
c2 += 1
<choose_model_class><EOS> 
n_normal_after = len(merged_df.query("PredictionString == @NORMAL"))
print(
 f"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}" 
)
 print(f"Keep {c0} Add {c1} Replace {c2}")
 submission_filepath = str("submission.csv")
 submission_df = merged_df[["image_id", "PredictionString"]]
submission_df.to_csv(submission_filepath, index=False)
 print(f"Saved to {submission_filepath}" )