Kaggle competition: RANZCR CLiP - Catheter and Line Position Challenge
<SOS> metric: MCAUC Kaggle data source: ranzcr-clip-catheter-and-line-position-challenge<load_from_csv>
import os
import random
import numpy as np
import pandas as pd
import cv2
import matplotlib.pyplot as plt
import torch
import torchvision
from torchvision import transforms
import tensorflow as tf
import albumentations as A
train_data = pd.read_csv(".. /input/covid19-global-forecasting-week-1/train.csv" )<count_missing_values>
DIR = ".. /input/ranzcr-clip-catheter-line-classification/"
train_data.isnull().sum()<rename_columns>
df_annotations = pd.read_csv(os.path.join(DIR, "train_annotations.csv"))
train_data = train_data.rename(columns = {"Province/State":"State" , "Country/Region":"Country" } )<count_missing_values>
row = df_annotations.iloc[8]
image_path = os.path.join(DIR, "train", row["StudyInstanceUID"] + ".jpg")
 chosen_image = cv2.imread(image_path )
train_data = train_data.fillna("Not Available")
 train_data.isnull().sum()<filter>
def NeedleAugmentation(image, n_needles=2, dark_needles=False, p=0.5, needle_folder='.. /input/xray-needle-augmentation'):
aug_prob = random.random()
if aug_prob < p:
height, width, _ = image.shape
needle_images = [im for im in os.listdir(needle_folder)if 'png' in im]
for _ in range(1, n_needles):
needle = cv2.cvtColor(cv2.imread(os.path.join(needle_folder, random.choice(needle_images))), cv2.COLOR_BGR2RGB)
 needle = cv2.flip(needle, random.choice([-1, 0, 1]))
needle = cv2.rotate(needle, random.choice([0, 1, 2]))
h_height, h_width, _ = needle.shape
roi_ho = random.randint(0, abs(image.shape[0] - needle.shape[0]))
roi_wo = random.randint(0, abs(image.shape[1] - needle.shape[1]))
roi = image[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]
img2gray = cv2.cvtColor(needle, cv2.COLOR_BGR2GRAY)
 ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)
 mask_inv = cv2.bitwise_not(mask)
 img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 if dark_needles:
img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 needle_fg = cv2.bitwise_and(img_bg, img_bg, mask=mask)
 else:
needle_fg = cv2.bitwise_and(needle, needle, mask=mask)
 dst = cv2.add(img_bg, needle_fg, dtype=cv2.CV_64F)
 image[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst
return image
train_data[train_data['ConfirmedCases']<0]<filter>
chosen_image = cv2.imread(image_path)
 aug_image = NeedleAugmentation(chosen_image, n_needles=3, dark_needles=False, p=1.0)
 plt.imshow(aug_image )
train_data[train_data['Fatalities']<0]<data_type_conversions>
chosen_image = cv2.imread(image_path)
 aug_image = NeedleAugmentation(chosen_image, n_needles=3, dark_needles=True, p=1.0)
 plt.imshow(aug_image )
train_data['Date'] = pd.to_datetime(train_data['Date'] )<filter>
torch_trans_list = [transforms.CenterCrop(( 178, 178)) ,
transforms.Resize(128),
transforms.RandomRotation(45),
transforms.RandomAffine(35),
transforms.RandomCrop(128),
transforms.RandomHorizontalFlip(p=1),
transforms.RandomPerspective(p=1),
transforms.RandomVerticalFlip(p=1)]
train_data[train_data['ConfirmedCases'] == train_data['ConfirmedCases'].max() ]<groupby>
INFER_MODE = 'TF'
date_country_total = train_data.groupby(['Date'] ).sum() 
<groupby>
!pip install.. /input/timm-package/timm-0.1.26-py3-none-any.whl
date_country_total['Total Cases'] = date_country_total['ConfirmedCases'].cumsum()<groupby>
if INFER_MODE == 'TORCH':
sys.path.append('.. /input/pytorch-image-models/pytorch-image-models-master')
 MODEL_DIR = '.. /input/ranzcr-pytorch-weights/'
OUTPUT_DIR = './'
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
 TEST_PATH = '.. /input/ranzcr-clip-catheter-line-classification/test'
country_cases = train_data.groupby(['Country' , 'Lat' , 'Long'] ).sum()
country_cases.reset_index(inplace=True)
 country_cases<data_type_conversions>
class CFG:
debug=False
num_workers=4
model_name='resnext50_32x4d'
size=600
batch_size=64
seed=42
target_size=11
target_cols=['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',
'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal',
'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',
'Swan Ganz Catheter Present']
n_fold=10
trn_fold=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
columns = train_data.columns.tolist()
columns
train_data['Date'] = pd.to_numeric(train_data['Date'] )<prepare_x_and_y>
if INFER_MODE == 'TORCH':
Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip,
RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout,
IAAAdditiveGaussianNoise, Transpose 
)
 warnings.filterwarnings('ignore')
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 def get_score(y_true, y_pred):
scores = []
for i in range(y_true.shape[1]):
score = roc_auc_score(y_true[:,i], y_pred[:,i])
 scores.append(score)
 avg_score = np.mean(scores)
 return avg_score, scores
def get_result(result_df):
preds = result_df[[f'pred_{c}' for c in CFG.target_cols]].values
labels = result_df[CFG.target_cols].values
score, scores = get_score(labels, preds)
 LOGGER.info(f'Score: {score:<.4f} Scores: {np.round(scores, decimals=4)}')
 @contextmanager
def timer(name):
t0 = time.time()
LOGGER.info(f'[{name}] start')
 yield
LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')
 def init_logger(log_file=OUTPUT_DIR+'inference.log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=log_file)
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
LOGGER = init_logger()
def seed_torch(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_torch(seed=CFG.seed)
 train = pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/train.csv')
 folds = train.copy()
Fold = GroupKFold(n_splits=CFG.n_fold)
 groups = folds['PatientID'].values
for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[CFG.target_cols], groups)) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int)
 display(folds.groupby('fold' ).size())
 test = pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/sample_submission.csv')
 if CFG.debug:
test = test.head()
class TestDataset(Dataset):
def __init__(self, df, transform=None):
self.df = df
self.file_names = df['StudyInstanceUID'].values
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
file_name = self.file_names[idx]
file_path = f'{TEST_PATH}/{file_name}.jpg'
image = cv2.imread(file_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if self.transform:
augmented = self.transform(image=image)
 image = augmented['image']
return image
def get_transforms(*, data):
if data == 'train':
return Compose([
Resize(CFG.size, CFG.size),
Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
),
ToTensorV2() ,
])
 elif data == 'valid':
return Compose([
Resize(CFG.size, CFG.size),
Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
),
ToTensorV2() ,
])
 class CustomResNext(nn.Module):
def __init__(self, model_name='resnext50_32x4d', pretrained=False):
super().__init__()
self.model = timm.create_model(model_name, pretrained=pretrained)
 n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, CFG.target_size)
 def forward(self, x):
x = self.model(x)
 return x
def inference(model, states, test_loader, device):
model.to(device)
 tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(images)in tk0:
images = images.to(device)
 avg_preds = []
for state in states:
model.load_state_dict(state['model'])
 model.eval()
with torch.no_grad() :
y_preds = model(images)
 avg_preds.append(y_preds.sigmoid().to('cpu' ).numpy())
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 probs = np.concatenate(probs)
 return probs
columns = [c for c in columns if c not in['Id','ConfirmedCases','Fatalities','State','Country']]
X_train = train_data[columns]
Y1_train = train_data['ConfirmedCases']
Y2_train = train_data['Fatalities']
print(X_train.shape)
 print(Y1_train.shape)
 print(Y2_train.shape )<train_model>
if INFER_MODE == 'TORCH':
model = CustomResNext(CFG.model_name, pretrained=False)
 states = [torch.load(MODEL_DIR+f'needle_more_augs_10folds_pretrained_{CFG.model_name}_fold{fold}_best.pth')for fold in CFG.trn_fold]
test_dataset = TestDataset(test, transform=get_transforms(data='valid'))
test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False,
num_workers=CFG.num_workers, pin_memory=True)
 predictions = inference(model, states, test_loader, device)
 test[CFG.target_cols] = predictions
test[['StudyInstanceUID'] + CFG.target_cols].to_csv(OUTPUT_DIR+'submission.csv', index=False )
regressor = RandomForestRegressor(n_estimators = 100 ,random_state=0)
 regressor.fit(X_train,Y1_train)
<load_from_csv>
!pip install /kaggle/input/kerasapplications -q
!pip install /kaggle/input/efficientnet-keras-source-code/ -q --no-deps
<data_type_conversions><EOS> 
if INFER_MODE == 'TF':
def auto_select_accelerator() :
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("Running on TPU:", tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print(f"Running on {strategy.num_replicas_in_sync} replicas")
 return strategy
def build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):
def decode(path):
file_bytes = tf.io.read_file(path)
 if ext == 'png':
img = tf.image.decode_png(file_bytes, channels=3)
 elif ext in ['jpg', 'jpeg']:
img = tf.image.decode_jpeg(file_bytes, channels=3)
 else:
raise ValueError("Image extension not supported")
 img = tf.cast(img, tf.float32)/ 255.0
img = tf.image.resize(img, target_size)
 return img
def decode_with_labels(path, label):
return decode(path), label
return decode_with_labels if with_labels else decode
def build_augmenter(with_labels=True):
def augment(img):
img = NeedleAugmentation(img, n_needles=2, dark_needles=False, p=0.5)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_flip_up_down(img)
 return img
def augment_with_labels(img, label):
return augment(img), label
return augment_with_labels if with_labels else augment
def build_dataset(paths, labels=None, bsize=32, cache=True,
decode_fn=None, augment_fn=None,
augment=True, repeat=True, shuffle=1024,
cache_dir=""):
if cache_dir != "" and cache is True:
os.makedirs(cache_dir, exist_ok=True)
 if decode_fn is None:
decode_fn = build_decoder(labels is not None)
 if augment_fn is None:
augment_fn = build_augmenter(labels is not None)
 AUTO = tf.data.experimental.AUTOTUNE
slices = paths if labels is None else(paths, labels)
 dset = tf.data.Dataset.from_tensor_slices(slices)
 dset = dset.map(decode_fn, num_parallel_calls=AUTO)
 dset = dset.cache(cache_dir)if cache else dset
dset = dset.map(augment_fn, num_parallel_calls=AUTO)if augment else dset
dset = dset.repeat() if repeat else dset
dset = dset.shuffle(shuffle)if shuffle else dset
dset = dset.batch(bsize ).prefetch(AUTO)
 return dset
COMPETITION_NAME = "ranzcr-clip-catheter-line-classification"
strategy = auto_select_accelerator()
BATCH_SIZE = strategy.num_replicas_in_sync * 16
IMSIZE =(224, 240, 260, 300, 380, 456, 528, 600)
 load_dir = f"/kaggle/input/{COMPETITION_NAME}/"
sub_df = pd.read_csv(load_dir + 'sample_submission.csv')
 test_paths = load_dir + "test/" + sub_df['StudyInstanceUID'] + '.jpg'
label_cols = sub_df.columns[1:]
test_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[7], IMSIZE[7]))
dtest = build_dataset(
 test_paths, bsize=BATCH_SIZE, repeat=False,
shuffle=False, augment=False, cache=False,
decode_fn=test_decoder 
)
 with strategy.scope() :
model = tf.keras.models.load_model(
 '.. /input/ranzcr-tpu-weights/efficientnetb7-tpu-needle-aug-resume-train.h5' 
)
 model.summary()
sub_df[label_cols] = model.predict(dtest, verbose=1)
 sub_df.to_csv('submission.csv', index=False )