Kaggle competition: RSNA STR Pulmonary Embolism Detection
<SOS> metric: weightedmeancolumnwiselogloss Kaggle data source: rsna-str-pulmonary-embolism-detection<predict_on_test>
!cp.. /input/gdcm-conda-install/gdcm.tar.
 !tar -xvzf gdcm.tar
!conda install --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2
pred_train1 = xgbmodel.predict(X_train)
 pred_test1 = xgbmodel.predict(X_test )<compute_test_metric>
sys.path = [
'.. /input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',
] + sys.path
print(np.sqrt(mean_squared_error(y_train, pred_train1)))
 print(np.sqrt(mean_squared_error(y_test, pred_test1)) )<predict_on_test>
import os
import cv2
import glob
import torch
import random
import pydicom
import datetime
import warnings
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from tqdm.notebook import tqdm
from torch.utils.data import Dataset
from scipy.ndimage.interpolation import zoom
from sklearn.model_selection import *
pred_xgb = xgbmodel.predict(test_X )<define_variables>
def seed_everything(seed):

 random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
test_id = test["id"]<create_dataframe>
SEED = 2020
seed_everything(SEED)
 MEAN = np.array([0.485, 0.456, 0.406])
 STD = np.array([0.229, 0.224, 0.225])
 DEVICE = torch.device("cuda")if torch.cuda.is_available() else torch.device("cpu")
 SIZE = 256
IMG_TARGET = "pe_present_on_image"
EXAM_TARGETS = [
"negative_exam_for_pe",
"rv_lv_ratio_gte_1",
"rv_lv_ratio_lt_1",
"leftsided_pe",
"chronic_pe",
"rightsided_pe",
"acute_and_chronic_pe",
"central_pe",
"indeterminate",
]
NUM_EXAM_TARGETS = len(EXAM_TARGETS )
pred_xgb = pd.DataFrame(np.exp(pred_xgb)-1,columns=["revenue"])
 pred_xgb<concatenate>
DATA_PATH = ".. /input/rsna-str-pulmonary-embolism-detection/"
sub=pd.concat([test_id, pred_xgb],axis=1 )<save_to_csv>
def normalizer(mean=MEAN, std=STD):
return albu.Compose([
albu.Normalize(mean=mean, std=std),
AT.transforms.ToTensor() ,
],
p=1,
)
sub.to_csv('TMDB_xgb.csv',index=False )<choose_model_class>
def to_jpg(img, name='img'):
cv2.imwrite(name + ".jpg", img)
 return cv2.imread(name + ".jpg" )
lgbmodel = lgb.LGBMRegressor(n_estimators=150,
objective='regression',
metric='rmse',
max_depth = 5,
num_leaves=30,
min_child_samples=30,
learning_rate=0.05,
boosting = 'gbdt',
min_data_in_leaf= 15,
feature_fraction = 0.9,
bagging_freq = 1,
bagging_fraction = 0.9,
importance_type='gain',
lambda_l1 = 0.2,
subsample=.8,
colsample_bytree=.8,
use_best_model=True )<train_model>
class PatientDataset(Dataset):

 def __init__(self, df, path, transforms=None):
self.df = df
self.path = path
self.img_paths = [sop + '.dcm' for sop in df['SOPInstanceUID'].values]
assert len(self.img_paths)== len(self.df)
 self.transforms = transforms
def __len__(self):
return len(self.img_paths)
 def __getitem__(self, idx):
image, z = dicom_to_img(self.path + self.img_paths[idx])
 image = to_jpg(image, name='tmp')
 if self.transforms:
image = self.transforms(image=image)["image"]
return image, z
lgbmodel.fit(X_train, y_train )<predict_on_test>
def load_model_weights(model, filename, verbose=1, cp_folder=""):

 if verbose:
print(f"
-> Loading weights from {os.path.join(cp_folder,filename)}
")
 try:
model.load_state_dict(os.path.join(cp_folder, filename), strict=False)
 except BaseException:
model.load_state_dict(
 torch.load(os.path.join(cp_folder, filename), map_location="cpu"),
strict=True, 
)
 return model
pred_train2 = lgbmodel.predict(X_train)
 pred_test2 = lgbmodel.predict(X_test )<compute_test_metric>
def add_ft_extractor(resnet):
resnet.extract_ft = lambda x: forward_extract_ft(resnet, x)
 def forward_extract_ft(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 ft = torch.flatten(x, 1)
 x = self.fc(ft)
 return x.view(-1), ft
def add_ft_extractor_enet(effnet):
effnet.forward = lambda x: forward_extract_ft_enet(effnet, x)
 def forward_extract_ft_enet(self, x):
x = self.extract_features(x)
 x = self._avg_pooling(x)
 ft = x.flatten(start_dim=1)
 x = self._dropout(ft)
 x = self._fc(x)
 return x.view(-1), ft
print(np.sqrt(mean_squared_error(y_train, pred_train2)))
 print(np.sqrt(mean_squared_error(y_test, pred_test2)) )<predict_on_test>
def define_model(name, num_classes=1):
if "resnext" in name:
model = resnext50_32x4d(pretrained=False)
 model.nb_ft = model.fc.in_features
model.fc = nn.Linear(model.nb_ft, num_classes)
 add_ft_extractor(model)
 else:
model = EfficientNet.from_name(name)
 model.nb_ft = model._fc.in_features
model._fc = nn.Linear(model.nb_ft, num_classes)
 add_ft_extractor_enet(model)
 return model
pred_lgb = lgbmodel.predict(test_X )<create_dataframe>
class RNNModel(nn.Module):
def __init__(self, ft_dim=2048, lstm_dim=64, dense_dim=256, use_msd=False, num_classes=9):
super().__init__()
self.use_msd = use_msd
self.mlp = nn.Sequential(
 nn.Linear(ft_dim, dense_dim * 2),
nn.ReLU() ,
nn.Linear(dense_dim * 2, dense_dim),
nn.ReLU() , 
)
 self.lstm = nn.LSTM(dense_dim, lstm_dim, batch_first=True, bidirectional=True)
 self.logits_exam = nn.Sequential(
 nn.Linear(lstm_dim * 4 + dense_dim * 2, lstm_dim),
nn.ReLU() ,
nn.Linear(lstm_dim, num_classes), 
)
 self.logits_img = nn.Sequential(
 nn.Linear(lstm_dim * 2 + dense_dim, lstm_dim),
nn.ReLU() ,
nn.Linear(lstm_dim, 1), 
)
 self.high_dropout = nn.Dropout(p=0.5)
 def forward(self, x):
features = self.mlp(x)
 features2, _ = self.lstm(features)
 features = torch.cat([features, features2], -1)
 mean = features.mean(1)
 max_, _ = features.max(1)
 pooled = torch.cat([mean, max_], -1)
 if self.use_msd and self.training:
logits_exam = torch.mean(
 torch.stack(
 [self.logits_exam(self.high_dropout(pooled)) for _ in range(5)],
dim=0, 
),
dim=0, 
)
 logits_img = torch.mean(
 torch.stack(
 [self.logits_img(self.high_dropout(features)) for _ in range(5)],
dim=0, 
),
dim=0, 
)
 else:
logits_exam = self.logits_exam(pooled)
 logits_img = self.logits_img(features)
 return logits_exam, logits_img.squeeze(-1 )
pred_lgb = pd.DataFrame(np.exp(pred_lgb)-1,columns=["revenue"])
 pred_lgb<concatenate>
def post_process(exam_pred, image_pred, eps=1e-6):
has_pe_image = image_pred.max() > 0.5 
(negative_exam_for_pe, rv_lv_ratio_gte_1, rv_lv_ratio_lt_1, leftsided_pe,
chronic_pe, rightsided_pe, acute_and_chronic_pe, central_pe, indeterminate)= exam_pred
broken = False
if has_pe_image:
if rv_lv_ratio_gte_1 < 0.5 and rv_lv_ratio_lt_1 < 0.5:
broken = True
max_ = np.max([leftsided_pe, rightsided_pe, central_pe])
 if max_ < 0.5:
broken = True
if not broken and has_pe_image:
if rv_lv_ratio_gte_1 > 0.5 and rv_lv_ratio_lt_1 > 0.5:
if rv_lv_ratio_gte_1 > rv_lv_ratio_lt_1:
rv_lv_ratio_lt_1 = 0.5 - eps
else:
rv_lv_ratio_gte_1 = 0.5 - eps
min_ = np.min([chronic_pe, acute_and_chronic_pe])
 if min_ > 0.5:
if chronic_pe > acute_and_chronic_pe:
acute_and_chronic_pe = 0.5 - eps
else:
chronic_pe = 0.5 - eps
if broken:
image_pred = np.clip(image_pred, 0, 0.5 - eps)
 assert image_pred.max() < 0.5
has_pe_image = image_pred.max() > 0.5
if not has_pe_image:
if indeterminate > 0.5 and negative_exam_for_pe > 0.5:
if indeterminate > negative_exam_for_pe:
negative_exam_for_pe = 0.5 - eps
else:
indeterminate = 0.5 - eps
elif indeterminate < 0.5 and negative_exam_for_pe < 0.5:
if indeterminate > negative_exam_for_pe:
indeterminate = 0.5 + eps
else:
negative_exam_for_pe = 0.5 + eps 
(rv_lv_ratio_gte_1, rv_lv_ratio_lt_1, leftsided_pe,
chronic_pe, rightsided_pe, acute_and_chronic_pe, central_pe)= np.clip(
 exam_pred[[1, 2, 3, 4, 5, 6, 7]],
0,
0.5 - eps 
)
 return np.array([
negative_exam_for_pe,
rv_lv_ratio_gte_1,
rv_lv_ratio_lt_1,
leftsided_pe,
chronic_pe,
rightsided_pe,
acute_and_chronic_pe,
central_pe,
indeterminate,
]), image_pred
sub1=pd.concat([test_id, pred_lgb],axis=1 )<save_to_csv>
def check_consistency(sub, test):

 for i in test['StudyInstanceUID'].unique() :
df_tmp = sub.loc[sub.id.str.contains(i, regex = False)].reset_index(drop = True)
 df_tmp['StudyInstanceUID'] = df_tmp['id'].str.split('_' ).str[0]
df_tmp['label_type'] = df_tmp['id'].str.split('_' ).str[1:].apply(lambda x: '_'.join(x))
del df_tmp['id']
if i == test['StudyInstanceUID'].unique() [0]:
df = df_tmp.copy()
else:
df = pd.concat([df, df_tmp], axis = 0)
 df_exam = df.pivot(index = 'StudyInstanceUID', columns = 'label_type', values = 'label')
 df_image = sub.loc[sub.id.isin(test.SOPInstanceUID)].reset_index(drop = True)
 df_image = df_image.merge(test, how = 'left', left_on = 'id', right_on = 'SOPInstanceUID')
 df_image.rename(columns = {"label": "pe_present_on_image"}, inplace = True)
 del df_image['id']
df = df_exam.merge(df_image, how = 'left', on = 'StudyInstanceUID')
 ids = ['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID']
labels = [c for c in df.columns if c not in ids]
df = df[ids + labels]
df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID'] ).pe_present_on_image.max())
 df_pos = df.loc[df.positive_images_in_exam > 0.5]
df_neg = df.loc[df.positive_images_in_exam <= 0.5]
rule1a = df_pos.loc[(( df_pos.rv_lv_ratio_lt_1 > 0.5)& 
(df_pos.rv_lv_ratio_gte_1 > 0.5)) | 
(( df_pos.rv_lv_ratio_lt_1 <= 0.5)& 
(df_pos.rv_lv_ratio_gte_1 <= 0.5)) ].reset_index(drop = True)
 rule1a['broken_rule'] = '1a'
rule1b = df_pos.loc[(df_pos.central_pe <= 0.5)& 
(df_pos.rightsided_pe <= 0.5)& 
(df_pos.leftsided_pe <= 0.5)].reset_index(drop = True)
 rule1b['broken_rule'] = '1b'
rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5)& 
(df_pos.chronic_pe > 0.5)].reset_index(drop = True)
 rule1c['broken_rule'] = '1c'
rule1d = df_pos.loc[(df_pos.indeterminate > 0.5)| 
(df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)
 rule1d['broken_rule'] = '1d'
rule2a = df_neg.loc[(( df_neg.indeterminate > 0.5)& 
(df_neg.negative_exam_for_pe > 0.5)) | 
(( df_neg.indeterminate <= 0.5)& 
(df_neg.negative_exam_for_pe <= 0.5)) ].reset_index(drop = True)
 rule2a['broken_rule'] = '2a'
rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1 > 0.5)| 
(df_neg.rv_lv_ratio_gte_1 > 0.5)| 
(df_neg.central_pe > 0.5)| 
(df_neg.rightsided_pe > 0.5)| 
(df_neg.leftsided_pe > 0.5)| 
(df_neg.acute_and_chronic_pe > 0.5)| 
(df_neg.chronic_pe > 0.5)].reset_index(drop = True)
 rule2b['broken_rule'] = '2b'
errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)
 print('Found', len(errors), 'inconsistent predictions')
 return errors
sub1.to_csv('TMDB_lgb.csv',index=False )<choose_model_class>
def extract_features(models, dataset, batch_size=32):
fts = np.empty(( 0, models[0].nb_ft))
zs = np.empty(0)
 preds = np.empty(0)
 loader = DataLoader(
 dataset,
batch_size=batch_size,
shuffle=False,
num_workers=1,
drop_last=False,
pin_memory=True 
)
 with torch.no_grad() :
for x, z in loader:
fts_tmp = []
preds_tmp = []
for model in models:
pred, ft = model(x.cuda())
 preds_tmp.append(torch.sigmoid(pred ).detach().cpu().numpy())
 fts_tmp.append(ft.detach().cpu().numpy())
 preds = np.concatenate([preds, np.mean(preds_tmp, 0)])
 fts = np.concatenate([fts, np.mean(fts_tmp, 0)])
 zs = np.concatenate([zs, z.numpy() ])
 order = np.argsort(zs)
 fts = fts[order]
preds = preds[order]
return preds, fts, order
catmodel = cat.CatBoostRegressor(iterations=2000,
learning_rate=0.01,
depth=8,
eval_metric='RMSE',
colsample_bylevel=0.8,
bagging_temperature = 0.2,
metric_period = None,
early_stopping_rounds=200 )<train_model>
def pred_to_sub(unique_df, df, pred_exams, pred_imgs, orders):
df_dic = {}
for i, study in enumerate(unique_df['StudyInstanceUID'].values):
reverse_order = np.argsort(orders[i])
 for t, target in enumerate(EXAM_TARGETS):
name = f'{study}_{target}'
label = pred_exams[i][t]
df_dic[name] = label
df_patient = df[df['StudyInstanceUID'] == study]
for s, sop in enumerate(df_patient['SOPInstanceUID'].values):
df_dic[sop] = pred_imgs[i][reverse_order[s]]
sub = pd.DataFrame.from_dict(df_dic, orient='index', columns=['label'] ).reset_index()
return sub.rename({'index': 'id'}, axis=1 )
catmodel.fit(X_train, y_train )<predict_on_test>
df_test = pd.read_csv(DATA_PATH + "test.csv" )
pred_train3 = catmodel.predict(X_train)
 pred_test3 = catmodel.predict(X_test )<compute_test_metric>
paths_test = [DATA_PATH + f"test/{study}/{series}/" for study, series in df_test[['StudyInstanceUID', 'SeriesInstanceUID']].values]
df_test['path'] = paths_test
print(np.sqrt(mean_squared_error(y_train, pred_train3)))
 print(np.sqrt(mean_squared_error(y_test, pred_test3)) )<predict_on_test>
unique_df_test = df_test[['path', 'StudyInstanceUID', 'SeriesInstanceUID']].drop_duplicates().reset_index(drop=True )
pred_cat = catmodel.predict(test_X )<create_dataframe>
CP_PATH = ".. /input/peweights/"
pred_cat = pd.DataFrame(np.exp(pred_cat)-1,columns=["revenue"])
 pred_cat<concatenate>
efficientnets = [f"efficientnet-b3__{i}.pt" for i in range(5)]
rnns_1 = [f'rnn_2_{i}.pt' for i in range(5)]
rnns_2 = [f'rnn_3_{i}.pt' for i in range(5)]
sub2=pd.concat([test_id, pred_cat],axis=1 )<save_to_csv>
first_lvl_models = []
for weights in efficientnets:
model = define_model('efficientnet-b3' ).cuda().eval()
model = load_model_weights(model, CP_PATH + weights)
 first_lvl_models.append(model )
sub2.to_csv('TMDB_cat.csv',index=False )<define_variables>
second_lvl_models = []
for weights in rnns_1:
model = RNNModel(
 ft_dim=1536,
lstm_dim=64,
use_msd=True,
).cuda().eval()
model = load_model_weights(model, CP_PATH + weights)
 second_lvl_models.append(model)
ansamble = 0.4 * pred_lgb["revenue"] + 0.2 * pred_xgb["revenue"] + 0.4 * pred_cat["revenue"]<concatenate>
IS_PUBLIC = len(unique_df_test)<= 650
DO_PUBLIC = True
DO_PRIVATE = True
DO_INFERENCE =(IS_PUBLIC and DO_PUBLIC)or(not IS_PUBLIC and DO_PRIVATE )
sub3=pd.concat([test_id, ansamble],axis=1 )<save_to_csv>
if DO_INFERENCE:
all_pred_exams = []
all_pred_imgs = []
orders = []
for path, study, series in tqdm(unique_df_test.values):
df_patient = df_test[df_test['StudyInstanceUID'] == study].reset_index(drop=True)
 dataset = PatientDataset(df_patient, path, normalizer())
 ')
 preds_1, fts, order = extract_features(first_lvl_models, dataset, batch_size=16)
 orders.append(order)
 ')
 pred_exams = []
pred_imgs = []
for model in second_lvl_models:
pred_exam, pred_img = inference_second_lvl(model, fts)
 pred_exams.append(pred_exam)
 pred_imgs.append(pred_img)
 pred_exams = np.mean(pred_exams, 0)
 pred_imgs = np.mean(pred_imgs, 0)
 pred_exams, pred_imgs = post_process(pred_exams, pred_imgs, eps=1e-6)
 all_pred_exams.append(pred_exams)
 all_pred_imgs.append(pred_imgs)
 sub = pred_to_sub(unique_df_test, df_test, all_pred_exams, all_pred_imgs, orders)
 else:
sub = pd.read_csv(DATA_PATH + "sample_submission.csv" )
sub3.to_csv('TMDB_ansamble.csv',index=False )<define_variables>
check_consistency(sub, df_test )
<concatenate><EOS> 
sub.to_csv('submission.csv', index=False)
 sub