Kaggle competition: RSNA STR Pulmonary Embolism Detection
<SOS> metric: weightedmeancolumnwiselogloss Kaggle data source: rsna-str-pulmonary-embolism-detection<count_missing_values>
!pip install.. /input/pytorch-image-models
if Path('.. /working/models' ).exists() :
shutil.rmtree('.. /working/models')
 shutil.copytree('.. /input/rsna2020-module/models', '.. /working/models')
 if Path('.. /working/functions' ).exists() :
shutil.rmtree('.. /working/functions')
 shutil.copytree('.. /input/rsna2020-module/functions', '.. /working/functions')
 print("OK!" )
train.isna().values.sum()<categorify>
!cp.. /input/gdcm-conda-install/gdcm.tar.
 !tar -xvzf gdcm.tar
!conda install --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2
print("OK!" )
def get_name(data):
try:
x = eval(data)
 return x[0]['name']
except:
return ''
def get_len(data):
try:
x = eval(data)
 return len(x)
 except:
return 0
def all_data(data, key = 'name'):
try:
x = eval(data)
 return ' '.join(sorted([i[key] for i in x]))
except:
return
def make_list(data, key = 'name'):
try:
data = eval(data)
 return [i[key] for i in data]
except:
return []
def apply_encode(data, name):
try:
if name in data:
return 1
else:
return 0
except:
return
def get_gender(data,index):
try:
data = eval(str(data))
count = 0
for i in data:
if i['gender'] == index:
count += 1
return count
except:
return 0
def singular_verb(feature_name):
if feature_name == 'production_countries':
return 'production_country'
elif feature_name == 'production_companies':
return 'production_company'
elif feature_name == 'spoken_languages':
return 'spoken_language'
elif feature_name == 'Keywords':
return 'Keyword'
else:
return feature_name
def fix_date(x):
year = x.split('/')[2]
if int(year)<= 19:
return x[:-2] + '20' + year
else:
return x[:-2] + '19' + year
def process_date(df):
date_parts = ["year", "weekday", "month", 'weekofyear', 'day', 'quarter']
for part in date_parts:
part_col = 'release_date' + "_" + part
df[part_col] = getattr(df['release_date'].dt, part ).astype(int)
 return df 
<data_type_conversions>
class RSNADataset(Dataset):
def __init__(
 self,
csv_path: str,
img_dir: str,
file_extension: str = 'dcm',
mode: str = 'test',
transform=None,
network_type: str = 'cnn_rnn', 
):
self.transform = transform
self.csv_path = Path(csv_path)
 self.img_dir = Path(img_dir)
 self.file_extension = file_extension
self.mode = mode
self.network_type = network_type
if Path('.. /input/rsna-str-pulmonary-embolism-detection/train' ).exists() and not DO_FULL:
df = pd.read_csv(self.csv_path ).head(200)
 else:
print("OK.Test Time Mode.")
 df = pd.read_csv(self.csv_path)
 df["file_name"] = df.SOPInstanceUID + '.' + self.file_extension
df["image_name"] = str(self.img_dir)+ '/' + \
df.StudyInstanceUID + '/' + df.SeriesInstanceUID + '/' + df.file_name
self.df = df
self.df["path_to_series_id"] = str(self.img_dir)+ '/' + \
self.df.StudyInstanceUID + '/' + self.df.SeriesInstanceUID
self.path_to_series_id = self.df["path_to_series_id"].unique()
def __len__(self):
return len(self.path_to_series_id)
 def __getitem__(self, index):
data_path = self.path_to_series_id[index]
dicoms, dicom_files = self._load_dicom_array(data_path)
 imgs = self._get_three_windowing_image(dicoms)
 if self.transform is not None:
imgs = imgs.transpose(0, 2, 3, 1)
 imgs = [self.transform(image=img ).transpose(
 2, 0, 1)for img in imgs]
imgs = np.stack(imgs)
 imgs = imgs.astype('float32')
 exam_level_name, image_level_name = self._get_file_names(dicom_files)
 return imgs, exam_level_name, image_level_name
def _load_dicom_array(self, path_to_series_id):
dicom_files = list(Path(path_to_series_id ).glob('*.dcm'))
dicoms = [pydicom.dcmread(d)for d in dicom_files]
slope = float(dicoms[0].RescaleSlope)
 intercept = float(dicoms[0].RescaleIntercept)
 z_pos = [float(d.ImagePositionPatient[-1])for d in dicoms]
dicoms = np.asarray([d.pixel_array for d in dicoms])
 dicoms = dicoms[np.argsort(z_pos)]
dicoms = dicoms * slope
dicoms = dicoms + intercept
dicom_files = np.array(dicom_files)[np.argsort(z_pos)]
return dicoms, dicom_files
def _windowing(self, img, window_length, window_width):
upper = window_length + window_width // 2
lower = window_length - window_width // 2
x = np.clip(img.copy() , lower, upper)
 x = x - np.min(x)
 x = x / np.max(x)
 x =(x * 255.0 ).astype('uint8')
 return x
def _get_three_windowing_image(self, dicoms):
img_lung = np.expand_dims(
 self._windowing(dicoms, -600, 1500), axis=1)
 img_mediastinal = np.expand_dims(
 self._windowing(dicoms, 40, 400), axis=1)
 img_pe_specific = np.expand_dims(
 self._windowing(dicoms, 100, 700), axis=1)
 return np.concatenate([
img_lung, img_pe_specific, img_mediastinal], axis=1)
 def _get_file_names(self, dicom_files):
exam_level_name = str(dicom_files[0].parent.parent.stem)
 dicom_files = dicom_files.tolist()
image_level_name = list(map(lambda x: str(x.stem), dicom_files))
return exam_level_name, image_level_name
train['runtime'].fillna(0, inplace=True)
 train['status'].fillna('Released', inplace=True)
 train['release_date'].fillna(train['release_date'].mode() [0], inplace=True)
 test['runtime'].fillna(0, inplace=True)
 test['status'].fillna('Released', inplace=True)
 test['release_date'].fillna(test['release_date'].mode() [0], inplace=True )<feature_engineering>
class RSNAAugmentation:
def __init__(self, size=512):
self.transform = A.Compose([
A.Resize(size, size, p=1.0),
])
 def __call__(self, **kwargs):
augmented = self.transform(**kwargs)
 img = augmented['image']
return img
train['collection_name'] = train['belongs_to_collection'].apply(get_name)
 train['num_of_collection'] = train['belongs_to_collection'].apply(get_len)
 train['num_of_genres'] = train['genres'].apply(get_len)
 train['num_of_countries'] = train['production_countries'].apply(get_len)
 train['num_of_companies'] = train['production_companies'].apply(get_len)
 train['num_of_spoken_languages'] = train['spoken_languages'].apply(get_len)
 train['num_of_cast'] = train['cast'].apply(get_len)
 train['num_of_crew'] = train['crew'].apply(get_len)
 train['num_of_keywords'] = train['Keywords'].apply(get_len)
 train['has_homepage'] = 0
train.loc[train['homepage'].isnull() == False, 'has_homepage'] = 1
test['collection_name'] = test['belongs_to_collection'].apply(get_name)
 test['num_of_collection'] = test['belongs_to_collection'].apply(get_len)
 test['num_of_genres'] = test['genres'].apply(get_len)
 test['num_of_countries'] = test['production_countries'].apply(get_len)
 test['num_of_companies'] = test['production_companies'].apply(get_len)
 test['num_of_spoken_languages'] = test['spoken_languages'].apply(get_len)
 test['num_of_cast'] = test['cast'].apply(get_len)
 test['num_of_crew'] = test['crew'].apply(get_len)
 test['num_of_keywords'] = test['Keywords'].apply(get_len)
 test['has_homepage'] = 0
test.loc[test['homepage'].isnull() == False, 'has_homepage'] = 1<feature_engineering>
def inference(config):
seed = config['seed']
torch.backends.cudnn.deterministic = True
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 n_tta = config['n_tta']
transform = RSNAAugmentation
dataset_args = {
'transform': transform(**config["transform"]),
**config["dataset"]
}
dataset = RSNADataset(**dataset_args)
 loader = DataLoader(dataset=dataset,
batch_size=config["batch_size"],
shuffle=False,
num_workers=config["n_workers"],
pin_memory=False)
 device = config['gpu'][0]
checkpoint_list = config['checkpoint']
net_list = []
for ckpt_path_dict in checkpoint_list:
model_args = {}
ckpt_path_cnn = Path(ckpt_path_dict['cnn'])
 cfg_path = ckpt_path_cnn.parents[1] / 'train_config.yaml'
with open(cfg_path, 'r', encoding='utf-8')as f:
cfg = yaml.load(f, Loader=yaml.SafeLoader)
 model_args['cnn_model'] = cfg["model"]["name"]
model_args['cnn_pretrained_path'] = ckpt_path_cnn
model_args['cnn_param'] = cfg["model"]["args"]
model_args['cnn_param']['pretrained'] = False
ckpt_path_cnn = Path(ckpt_path_dict['rnn'])
 cfg_path = ckpt_path_cnn.parents[1] / 'train_config.yaml'
with open(cfg_path, 'r', encoding='utf-8')as f:
cfg = yaml.load(f, Loader=yaml.SafeLoader)
 model_args['rnn_model'] = cfg["model"]["name"]
model_args['rnn_pretrained_path'] = ckpt_path_cnn
model_args['rnn_param'] = cfg["model"]["args"]
net = getattr(models, 'CNN_RNN' )(**model_args)
 net.to(device)
 net.eval()
net_list.append(net)
 chunk_size_list = config['chunk_size_list']
if len(chunk_size_list)== 1:
chunk_size_list = chunk_size_list * len(net_list)
 assert len(chunk_size_list)== len(net_list)
 tta = None
exam_names = []
image_names = []
with torch.no_grad() :
results_image_level = []
results_exam_level = []
for x, exam_name, image_name in tqdm(loader):
exam_names.append(exam_name[0])
 image_name = list(map(lambda x: x[0], image_name))
image_names.extend(image_name)
 n_sequence = x.size() [1]
result_tta_image, result_tta_exam = [], []
for tta_cnt in range(n_tta):
image = x.clone()
result_net_exam, result_net_image = [], []
for net_cnt, net in enumerate(net_list):
embeddings = []
for i in range(0, n_sequence, chunk_size_list[net_cnt]):
embedding = net.cnn(image[:, i:i + chunk_size_list[net_cnt], :, :, :].to(device))
embeddings.append(embedding)
 embeddings = torch.cat(embeddings, dim=1)
 image_level, exam_level = net.rnn(embeddings)
 image_level = torch.sigmoid(
 image_level ).cpu().detach().numpy().reshape(-1)
 exam_level = torch.sigmoid(
 exam_level ).cpu().detach().numpy().reshape(-1)
 exam_level = label_consistency(image_level, exam_level)
 result_net_image.append(image_level)
 result_net_exam.append(exam_level)
 result_net_image = np.array(result_net_image)
 result_tta_image.append(result_net_image)
 result_net_exam = np.array(result_net_exam)
 result_tta_exam.append(result_net_exam)
 result_tta_image = np.array(result_tta_image)
 results_image_level.append(result_tta_image)
 result_tta_exam = np.array(result_tta_exam)
 results_exam_level.append(result_tta_exam)
 results_exam_level = np.array(results_exam_level)
 results_exam_level = results_exam_level.mean(axis=1 ).mean(axis=1)
 results_image_level = list(map(
 lambda x: x.mean(axis=0 ).mean(axis=0), results_image_level))
results_exam_level = np.stack([label_consistency(image_level, exam_level)
 for image_level, exam_level in zip(results_image_level, results_exam_level)])
 results_image_level = np.concatenate(results_image_level)
 results_exam_level = results_exam_level.reshape(-1)
 exam_names = get_exam_names(exam_names)
 assert len(image_names)== len(results_image_level)
 assert len(exam_names)== len(results_exam_level)
 names = image_names + exam_names
results = np.concatenate([results_image_level, results_exam_level])
 submission = pd.DataFrame([names, results], index=['id', 'label'] ).T
if check_consistency(submission, dataset.df):
print("Great! Fanstastic! You are genious!!!")
 submission.to_csv('submission.csv', index=False)
 else:
print("ERROR! submission file doesn't satisfy concistency!!")
 def label_consistency(image_level, exam_level):
p_negative_exam_for_pe = exam_level[0]
p_indeterminate = exam_level[1]
p_chronic_pe = exam_level[2]
p_acute_and_chronic_pe = exam_level[3]
p_central_pe = exam_level[4]
p_leftsided_pe = exam_level[5]
p_rightsided_pe = exam_level[6]
p_rv_lv_ratio_gte_1 = exam_level[7]
p_rv_lv_ratio_lt_1 = exam_level[8]
pe_exist = np.any(image_level > 0.5)
 if pe_exist:
p_negative_exam_for_pe = np.clip(p_negative_exam_for_pe, None, 0.499)
 p_indeterminate = np.clip(p_indeterminate, None, 0.499)
 if p_chronic_pe > 0.5 and p_acute_and_chronic_pe > 0.5:
tmp_list = [p_chronic_pe, p_acute_and_chronic_pe]
tmp_list[np.argmin(tmp_list)] = 0.499
p_chronic_pe, p_acute_and_chronic_pe = tmp_list
if p_central_pe <= 0.5 and p_leftsided_pe <= 0.5 and p_rightsided_pe <= 0.5:
tmp_list = [p_central_pe, p_leftsided_pe, p_rightsided_pe]
tmp_list[np.argmax(tmp_list)] = 0.501
p_central_pe, p_leftsided_pe, p_rightsided_pe = tmp_list
if p_rv_lv_ratio_gte_1 <= 0.5 and p_rv_lv_ratio_lt_1 <= 0.5:
tmp_list = [p_rv_lv_ratio_gte_1, p_rv_lv_ratio_lt_1]
tmp_list[np.argmax(tmp_list)] = 0.501
p_rv_lv_ratio_gte_1, p_rv_lv_ratio_lt_1 = tmp_list
if p_rv_lv_ratio_gte_1 > 0.5 and p_rv_lv_ratio_lt_1 > 0.5:
tmp_list = [p_rv_lv_ratio_gte_1, p_rv_lv_ratio_lt_1]
tmp_list[np.argmin(tmp_list)] = 0.499
p_rv_lv_ratio_gte_1, p_rv_lv_ratio_lt_1 = tmp_list
else:
if p_negative_exam_for_pe <= 0.5 and p_indeterminate <= 0.5:
tmp_list = [p_negative_exam_for_pe, p_indeterminate]
tmp_list[np.argmax(tmp_list)] = 0.501
p_negative_exam_for_pe, p_indeterminate = tmp_list
if p_negative_exam_for_pe > 0.5 and p_indeterminate > 0.5:
tmp_list = [p_negative_exam_for_pe, p_indeterminate]
tmp_list[np.argmin(tmp_list)] = 0.499
p_negative_exam_for_pe, p_indeterminate = tmp_list
p_chronic_pe = np.clip(p_chronic_pe, None, 0.499)
 p_acute_and_chronic_pe = np.clip(p_acute_and_chronic_pe, None, 0.499)
 p_central_pe = np.clip(p_central_pe, None, 0.499)
 p_leftsided_pe = np.clip(p_leftsided_pe, None, 0.499)
 p_rightsided_pe = np.clip(p_rightsided_pe, None, 0.499)
 p_rv_lv_ratio_gte_1 = np.clip(p_rv_lv_ratio_gte_1, None, 0.499)
 p_rv_lv_ratio_lt_1 = np.clip(p_rv_lv_ratio_lt_1, None, 0.499)
 exam_level = np.array([
p_negative_exam_for_pe,
p_indeterminate,
p_chronic_pe,
p_acute_and_chronic_pe,
p_central_pe,
p_leftsided_pe,
p_rightsided_pe,
p_rv_lv_ratio_gte_1,
p_rv_lv_ratio_lt_1,
])
 return exam_level
def get_exam_names(exam_names):
target_cols = [
'negative_exam_for_pe',
'indeterminate',
'chronic_pe', 'acute_and_chronic_pe',
'central_pe', 'leftsided_pe', 'rightsided_pe',
'rv_lv_ratio_gte_1', 'rv_lv_ratio_lt_1',
]
new_exam_names = []
for e in exam_names:
for col in target_cols:
new_exam_names.append(e + '_' + col)
 return new_exam_names
def check_consistency(sub, test_csv):
str_split = sub.id.str.split('_', 1, expand=True)
 str_split.columns = ['StudyInstanceUID', 'label_type']
condition = ~str_split.label_type.isnull()
new_df = pd.concat([sub[condition], str_split[condition]], axis=1)
 del new_df['id']
df_exam = new_df.pivot(index='StudyInstanceUID', columns='label_type', values='label')
 condition = str_split.label_type.isnull()
df_image = sub[condition]
df_image = df_image.merge(test_csv, how='left', left_on='id', right_on='SOPInstanceUID')
 df_image.rename(columns = {"label": "pe_present_on_image"}, inplace=True)
 del df_image['id']
df = df_exam.merge(df_image, how='left', on='StudyInstanceUID')
 ids = ["StudyInstanceUID", "SeriesInstanceUID", "SOPInstanceUID"]
labels = [c for c in df.columns if c not in ids]
df = df[ids + labels]
df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID'] ).pe_present_on_image.max())
 df_pos = df.loc[df.positive_images_in_exam > 0.5]
df_neg = df.loc[df.positive_images_in_exam <= 0.5]
rule1a = df_pos.loc[(( df_pos.rv_lv_ratio_lt_1 > 0.5)& 
(df_pos.rv_lv_ratio_gte_1 > 0.5)) | 
(( df_pos.rv_lv_ratio_lt_1 <= 0.5)& 
(df_pos.rv_lv_ratio_gte_1 <= 0.5)) ].reset_index(drop = True)
 rule1a['broken_rule'] = '1a'
rule1b = df_pos.loc[(df_pos.central_pe <= 0.5)& 
(df_pos.rightsided_pe <= 0.5)& 
(df_pos.leftsided_pe <= 0.5)].reset_index(drop = True)
 rule1b['broken_rule'] = '1b'
rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5)& 
(df_pos.chronic_pe > 0.5)].reset_index(drop = True)
 rule1c['broken_rule'] = '1c'
rule1d = df_pos.loc[(df_pos.indeterminate > 0.5)| 
(df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)
 rule1d['broken_rule'] = '1d'
rule2a = df_neg.loc[(( df_neg.indeterminate > 0.5)& 
(df_neg.negative_exam_for_pe > 0.5)) | 
(( df_neg.indeterminate <= 0.5)& 
(df_neg.negative_exam_for_pe <= 0.5)) ].reset_index(drop = True)
 rule2a['broken_rule'] = '2a'
rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1 > 0.5)| 
(df_neg.rv_lv_ratio_gte_1 > 0.5)| 
(df_neg.central_pe > 0.5)| 
(df_neg.rightsided_pe > 0.5)| 
(df_neg.leftsided_pe > 0.5)| 
(df_neg.acute_and_chronic_pe > 0.5)| 
(df_neg.chronic_pe > 0.5)].reset_index(drop = True)
 rule2b['broken_rule'] = '2b'
errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)
 if len(errors)== 0:
return True
else:
return False
<feature_engineering><EOS> 
config = {}
config['checkpoint'] = [
{
'cnn': '/kaggle/input/rsna2020-models/run-20201020_060628-2cpwvzq3/files/checkpoints/epoch01-val_loss0.095.ckpt',
'rnn': '/kaggle/input/rsna2020-models/run-20201021_135533-33iuaotu/files/checkpoints/epoch32-val_loss0.145.ckpt',
},
{
'cnn': '/kaggle/input/rsna2020-models/run-20201020_150752-28onzftp/files/checkpoints/epoch02-val_loss0.095.ckpt',
'rnn': '/kaggle/input/rsna2020-models/run-20201021_135644-3kch48tn/files/checkpoints/epoch28-val_loss0.146.ckpt',
},
{
'cnn': '/kaggle/input/rsna2020-models/run-20201020_151143-31kxh5r3/files/checkpoints/epoch02-val_loss0.093.ckpt',
'rnn': '/kaggle/input/rsna2020-models/run-20201021_135714-3bjhhbju/files/checkpoints/epoch31-val_loss0.143.ckpt',
},
{
'cnn': '/kaggle/input/rsna2020-models/run-20201022_070959-21nvi91y/files/checkpoints/epoch02-val_loss0.027.ckpt',
'rnn': '/kaggle/input/rsna2020-models/run-20201023_041844-1y4pdiyr/files/checkpoints/epoch36-val_loss0.150.ckpt',
},
{
'cnn': '/kaggle/input/rsna2020-models/run-20201022_071013-3iwenrxd/files/checkpoints/epoch02-val_loss0.027.ckpt',
'rnn': '/kaggle/input/rsna2020-models/run-20201023_041955-1fmrfriu/files/checkpoints/epoch34-val_loss0.150.ckpt',
},
{
'cnn': '/kaggle/input/rsna2020-models/run-20201022_070525-19vyc6gd/files/checkpoints/epoch02-val_loss0.026.ckpt',
'rnn': '/kaggle/input/rsna2020-models/run-20201023_042016-30e7wvxy/files/checkpoints/epoch34-val_loss0.141.ckpt',
},
]
config['batch_size'] = 1
config['chunk_size_list'] = [
64,
64,
64,
64,
64,
64,
]
config['n_workers'] = 1
config['n_tta'] = 1
config['seed'] = 42
config['gpu'] = [0]
config['transform'] = {
'size': 512
}
config['dataset'] = {
"csv_path": "/kaggle/input/rsna-str-pulmonary-embolism-detection/test.csv",
"img_dir": "/kaggle/input/rsna-str-pulmonary-embolism-detection/test",
"file_extension": "dcm",
"mode": "test",
"network_type": "cnn_rnn",
}
DO_FULL = False