Kaggle competition: RANZCR CLiP - Catheter and Line Position Challenge
<SOS> metric: MCAUC Kaggle data source: ranzcr-clip-catheter-and-line-position-challenge<save_to_csv>
import tensorflow as tf
from tensorflow.keras import layers
import os
import re
import math
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
df.to_csv('submission.csv', index=False )<load_from_csv>
GCS_DS_PATH = ".. /input/ranzcr-clip-catheter-line-classification"
base_path = '/kaggle/input/Kannada-MNIST/'
train = pd.read_csv(base_path + 'train.csv')
 test = pd.read_csv(base_path + 'test.csv')
 dig_mnist = pd.read_csv(base_path + 'Dig-MNIST.csv')
 train_label = train['label']
train_data_raw = train.drop('label', axis=1, inplace=False)
 train_data = []
train_data_raw = np.array(train_data_raw)
 for i in train_data_raw:
image = i.reshape(28,28,1)
 train_data.append(image)
 test.drop('id', axis=1, inplace=True)
 test_data_raw = np.array(test)
 test_data = []
for i in test_data_raw:
image = i.reshape(28,28,1)
 test_data.append(image)
<prepare_x_and_y>
train_df = pd.read_csv(GCS_DS_PATH+"/train.csv")
 train_df.index = train_df["StudyInstanceUID"]
del train_df["StudyInstanceUID"]
train_annot_df = pd.read_csv(GCS_DS_PATH+"/train_annotations.csv")
 train_annot_df.index = train_annot_df["StudyInstanceUID"]
del train_annot_df["StudyInstanceUID"]
train_data = np.array(train_data)/255.0
test_data = np.array(test_data)/255.0
train_label = np.array(train_label ).reshape(60000,-1 )<choose_model_class>
classes = list(train_df.columns[:-1])
 classes_normal= [name for name in classes[:-1] if name.split(" - ")[1] == "Normal"]
classes_abnormal= [name for name in classes[:-1] if name.split(" - ")[1] == "Abnormal"]
classes_borderline = [name for name in classes[:-1] if name.split(" - ")[1] == "Borderline"]
classes_count = train_df[classes].sum(axis = 0)
 num_classes = len(classes_count)
 print("Number of Classes: {}".format(num_classes))
classes_count
es = EarlyStopping(monitor='accuracy', mode='min', verbose=1, patience=5,baseline=0.99)
 def define_model() :
model2 = keras.models.Sequential([
keras.layers.Conv2D(16,(3,3), input_shape=(28,28,1), activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Conv2D(16,(5,5), activation='relu', padding='same'),
keras.layers.BatchNormalization() ,
keras.layers.MaxPooling2D(2,2),
keras.layers.Dropout(0.4),
keras.layers.Conv2D(32,(3,3), activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Conv2D(32,(5,5), activation='relu', padding='same'),
keras.layers.BatchNormalization() ,
keras.layers.MaxPooling2D(2,2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
keras.layers.Conv2D(64,(3,3), activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Conv2D(64,(5,5), activation='relu', padding='same'),
keras.layers.BatchNormalization() ,
keras.layers.MaxPooling2D(2,2),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
keras.layers.Flatten() ,
keras.layers.Dense(128),
keras.layers.BatchNormalization() ,
keras.layers.Activation('relu'),
keras.layers.Dropout(0.6),
keras.layers.Dense(64),
keras.layers.BatchNormalization() ,
keras.layers.Activation('relu'),
keras.layers.Dropout(0.5),
keras.layers.Dense(10, activation='softmax')
 ])
 return model2
model = define_model()
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
 model.summary()<choose_model_class>
class_weights = {}
ls = list(classes_count.values)
 tot_samples = sum(ls)
 for i in range(num_classes):
class_weights[i] = tot_samples/(num_classes*ls[i])
 class_weights
def define_model_ResNet50() :
model = keras.models.Sequential()
input_layer = keras.layers.Input(shape=(224, 224, 3), name='image_input')
 model.add(DenseNet121(weights=None, include_top=False, input_tensor=input_layer))
model.add(keras.layers.Flatten())
 model.add(keras.layers.Dense(128))
model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.Activation('relu'))
model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.Dense(32))
model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.Activation('relu'))
model.add(keras.layers.Dropout(0.1))
model.add(keras.layers.Dense(10, activation='softmax'))
model.layers[0].trainable = True
model.summary()
return model 
<train_model>
patient_ids = train_df["PatientID"].unique()
patientwise_count = train_df['PatientID'].value_counts()
num_patients = len(patientwise_count)
 print("Number of patients: ",num_patients)
 patientwise_count
EPOCHS =100
X_train ,X_test,Y_train,Y_test= train_test_split(train_data,train_label,test_size=0.1)
 print(np.array(Y_train ).shape)
 es = keras.callbacks.EarlyStopping(monitor='accuracy', mode='min', patience=3, baseline=0.99)
 checkpoint = ModelCheckpoint('best_weights.h5', monitor='val_loss', sava_best_only=True, mode='auto', period=1)
 model = define_model()
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
<load_from_csv>
IMAGE_SIZE = [600,600]
AUTO = tf.data.experimental.AUTOTUNE
TEST_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + '/test_tfrecords/*.tfrec' )
<prepare_x_and_y>
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, [*IMAGE_SIZE])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"StudyInstanceUID" : tf.io.FixedLenFeature([], tf.string),
"image" : tf.io.FixedLenFeature([], tf.string),
"ETT - Abnormal" : tf.io.FixedLenFeature([], tf.int64),
"ETT - Borderline" : tf.io.FixedLenFeature([], tf.int64),
"ETT - Normal" : tf.io.FixedLenFeature([], tf.int64),
"NGT - Abnormal" : tf.io.FixedLenFeature([], tf.int64),
"NGT - Borderline" : tf.io.FixedLenFeature([], tf.int64),
"NGT - Incompletely Imaged" : tf.io.FixedLenFeature([], tf.int64),
"NGT - Normal" : tf.io.FixedLenFeature([], tf.int64),
"CVC - Abnormal" : tf.io.FixedLenFeature([], tf.int64),
"CVC - Borderline" : tf.io.FixedLenFeature([], tf.int64),
"CVC - Normal" : tf.io.FixedLenFeature([], tf.int64),
"Swan Ganz Catheter Present" : tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = [example['ETT - Abnormal'],
example['ETT - Borderline'],
example['ETT - Normal'],
example['NGT - Abnormal'],
example['NGT - Borderline'],
example['NGT - Incompletely Imaged'],
example['NGT - Normal'],
example['CVC - Abnormal'],
example['CVC - Borderline'],
example['CVC - Normal'],
example['Swan Ganz Catheter Present']]
label = [tf.cast(i,tf.float32)for i in label]
return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"StudyInstanceUID" : tf.io.FixedLenFeature([], tf.string),
"image" : tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['StudyInstanceUID']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
<load_from_csv>
def data_augment(image, label):
image = tf.image.random_flip_left_right(image)
 return image,label
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 print('Dataset: {} unlabeled test images'.format(NUM_TEST_IMAGES))
<prepare_x_and_y>
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
test_ds = get_test_dataset()
print("Test:", test_ds )
print(np.array(X_train ).shape)
 print(np.array(Y_train ).shape)
 datagen = ImageDataGenerator(featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=20,
zoom_range = 0.2,
width_shift_range=0.20,
height_shift_range=0.20,
horizontal_flip=False,
vertical_flip=False)
 history = model.fit_generator(datagen.flow(X_train,Y_train),
epochs = EPOCHS, validation_data =(X_test,Y_test),
verbose = 1, callbacks=[es,checkpoint] )<load_pretrained>
!pip install /kaggle/input/kerasapplications -q
!pip install /kaggle/input/efficientnet-keras-source-code/ -q --no-deps
model_prediction = define_model()
model_prediction.load_weights('best_weights.h5' )<data_type_conversions>
model = tf.keras.models.load_model(".. /input/ranzcr-clip-tpu/model.h5" )
<save_to_csv><EOS> 
test_ids=[]
test_pred = []
j=0
for batch in test_ds:
images,ids_batch = batch
pred_batch = model.predict(images)
 for i,ids in enumerate(ids_batch):
j+=1
if j%500 == 0:
print(str(j),"Test Images Done")
 test_ids.append(ids)
 test_pred.append(pred_batch[i])
 test_ids = [np.array(i ).astype("str" ).tolist() for i in test_ids]
test_df = pd.DataFrame(test_pred,index=test_ids,columns=classes)
 test_df.index.name = "StudyInstanceUID"
test_df.to_csv("./submission.csv" )