Kaggle competition: RSNA STR Pulmonary Embolism Detection
<SOS> metric: weightedmeancolumnwiselogloss Kaggle data source: rsna-str-pulmonary-embolism-detection<categorify>
%%capture
!pip install.. /input/rsnapkgs/timm-0.2.2-py3-none-any.whl
!pip install.. /input/pytorch-transformers/pytorch_transformers-1.2.0-py3-none-any.whl
!cp.. /input/gdcm-conda-install/gdcm.tar.
 !tar -xvzf gdcm.tar
!conda install --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2
print("done" )
def count_word_list(series):
len_max = series.apply(len ).max()
tmp = series.map(lambda x: x+["nashi"]*(len_max-len(x)))
 word_set = set(sum(list(series.values), []))
for n in range(len_max):
word_dfn = pd.get_dummies(tmp.apply(lambda x: x[n]))
word_dfn = word_dfn.reindex(word_set, axis=1 ).fillna(0 ).astype(int)
 if n==0:
word_df = word_dfn
else:
word_df = word_df + word_dfn
return word_df<define_variables>
PATH = '.. /input/rsnasubcheck'
sys.path.append(PATH)
 PATH = '.. /input/rsnastr2020/rsnastr'
sys.path.append(PATH)
 np.set_printoptions(suppress=True)
 logger = get_logger('Kaggle', 'INFO')
dfdic_feature = {}<feature_engineering>
class args:
device = 'cuda' if torch.cuda.is_available() else 'cpu'
batchsize = 1
imgbatchsize = 128
folds = [0,1,2]
load_train = False
do_full = True
delta = False
df["genre_names"] = df["genres"].apply(lambda x : [ i["name"] for i in x] )<feature_engineering>
def dict2class(cfg):
cfg['device'] = args.device
cfg = namedtuple('Struct', cfg.keys() )(*cfg.values())
 return cfg
cfgimg = dict2class(load_config(f'{PATH}/configs/effnetb5_lr5e4_multi.json'))
cfgseq1 = dict2class(load_config(f'{PATH}/configs/b5_seq_lstm.json', defaults=RSNA_CFG))
cfgseq2 = dict2class(load_config(f'{PATH}/configs/b5_seq_transformer_1layer.json', defaults=RSNA_CFG))
cfgseq3 = dict2class(load_config(f'{PATH}/configs/b5_seq_transformer_2layer.json', defaults=RSNA_CFG))
df['num_genres'] = df['genres'].apply(lambda x: len(x)if x != {} else 0 )<drop_column>
class RSNAImageSequenceDataset(Dataset):
def __init__(self,
datadf,
folddf,
mode="train"):
self.mode = mode
self.datadf = datadf
self.folddf = folddf
self.imgclasses = CFG['image_target_cols']
self.studyclasses = CFG['exam_target_cols']
def __len__(self):
return len(self.folddf)
 def __getitem__(self, idx):
studyidx = self.folddf.iloc[idx].StudyInstanceUID
seriesidx = self.folddf.iloc[idx].SeriesInstanceUID
studydf = self.datadf.query('StudyInstanceUID == @studyidx')\ 
.query('SeriesInstanceUID == @seriesidx')
 imgs = []
imgnames = []
imglabels = []
if self.mode == 'train':
studylabels = studydf[self.studyclasses].iloc[0].values
for i, samp in studydf.reset_index().iterrows() :
if self.mode == 'train':
imglabels.append(samp['pe_present_on_image'])
 sampkey, img_name = self.image_file(samp)
 try:
dicom_object = pydicom.read_file(img_name)
 img = ip_window(dicom_object)
 except:
img = np.zeros(shape=(512,512,3), dtype=np.uint8)
 imgs.append(img)
 imgnames.append(sampkey)
 imgs = np.stack(imgs)
 if self.mode == 'train':
labels = torch.tensor(imglabels)
 return {'img_name': imgnames, 'image': imgs, 'imglabels': imglabels, 'studylabels': studylabels}
return {'img_name': imgnames, 'image': imgs}
def image_file(self, samp):
sampkey = '/'.join([samp.StudyInstanceUID, samp.SeriesInstanceUID,samp.SOPInstanceUID])
 imgname = os.path.join('.. /input/rsna-str-pulmonary-embolism-detection',
self.mode,
sampkey)+ '.dcm'
return sampkey, imgname
def collateimgfn(batch):
maxlen = max([l['image'].shape[0] for l in batch])
 for b in batch:
masklen = maxlen-len(b['image'])
 b['mask'] = np.ones(( maxlen))
b['mask'][:masklen] = 0
outbatch = {'image': np.concatenate([b['image'] for b in batch], 0)}
outbatch['mask'] = torch.tensor(np.vstack([np.expand_dims(b['mask'], 0)\
for b in batch])).float()
outbatch['img_name'] = []
for b in batch:
outbatch['img_name'] += b['img_name']
if 'imglabels' in batch[0]:
for b in batch:
masklen = maxlen-len(b['image'])
 b['imglabelsout'] = np.ones(( maxlen)) *-1
b['imglabelsout'][masklen:] = b['imglabels']
outbatch['imglabels'] = torch.tensor(np.vstack([np.expand_dims(b['imglabelsout'], 0)\
for b in batch])).long()
outbatch['studylabels'] = torch.tensor(np.vstack([np.expand_dims(b['studylabels'], 0)\
for b in batch])).long()
return outbatch
def batchPredsfn(studypreds, imgpreds, imgnames):
preds = []
batchdf = pd.DataFrame([i.split('/')for i in imgnames],
columns = ['StudyInstanceUID', 'SeriesInstanceUID','SOPInstanceUID'])
 studies = batchdf.StudyInstanceUID.unique().tolist()
for study, row in zip(studies, studypreds):
for col, pred in zip(CFG['exam_target_cols'], row):
preds.append([f'{study}_{col}', pred])
 for nm, pred in zip(batchdf.SOPInstanceUID, imgpreds):
preds.append([nm, pred])
 return preds
def aug_batch(imgs, mean_, std_, device):
imgs = imgs.to(device ).float()
return(((imgs / 255)- mean_)/ std_ ).permute(0, 3, 1, 2 ).half()
dfdic_feature["genre"] = count_word_list(df["genre_names"])
 dfdic_feature["genre"] = dfdic_feature["genre"].drop("TV Movie", axis=1)
 dfdic_feature["genre"].head()<feature_engineering>
!cp.. /input/rsna-str-pulmonary-embolism-detection/sample_submission.csv submission.csv
if args.load_train:
trndf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/train.csv')
 if os.path.exists('.. /input/rsna-str-pulmonary-embolism-detection/train')and not args.do_full:
tstdf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/test.csv' ).head(2000)
 else:
tstdf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/test.csv')
 keycols = ['StudyInstanceUID', 'SeriesInstanceUID']
DEBUG =(tstdf.shape[0]==146853)
 DEBUG
n_language = df.loc[:train.index[-1], "original_language"].value_counts()
large_language = n_language[n_language>=20].index
df.loc[~df["original_language"].isin(large_language), "original_language"] = "small"<data_type_conversions>
loaderargs = {'num_workers' : 4, 'pin_memory': False, 'drop_last': False, 'collate_fn' : collateimgfn}
if args.load_train:
trndataset = RSNAImageSequenceDataset(mode = 'train',
folddf = trndf[keycols].drop_duplicates().reset_index(drop=True),
datadf = trndf)
 trnloader = DataLoader(trndataset, batch_size=args.batchsize, shuffle=False, **loaderargs)
 tstdataset = RSNAImageSequenceDataset(mode = 'test',
folddf = tstdf[keycols].drop_duplicates().reset_index(drop=True),
datadf = tstdf)
 tstloader = DataLoader(tstdataset, batch_size=args.batchsize, shuffle=False, **loaderargs )
df["original_language"] = df["original_language"].astype("category" )<feature_engineering>
def load_image_model(wtsnm, conf, device = args.device):
nclasses = len(conf.image_target_cols)+ len(conf.exam_target_cols)
 model = classifiers.__dict__[conf.network](encoder=f"{conf.encoder}_infer",
nclasses = nclasses,
infer = True)
 checkpoint = torch.load(wtsnm, map_location=torch.device(device))
model.load_state_dict(checkpoint['state_dict'])
 model = model.half().to(device)
 model = model.eval()
return model
def load_exam_model(wtsnm, cfg, embed_size = 2048, device = args.device):
model = LSTMNet(embed_size,
nimgclasses = len(cfg.image_target_cols),
nstudyclasses = len(cfg.exam_target_cols),
LSTM_UNITS=cfg.lstm_units,
DO = cfg.dropout)
 checkpoint = torch.load(wtsnm, map_location=torch.device(device))
model.load_state_dict(checkpoint)
 model = model.half().to(device)
 model = model.eval()
return model
def load_exam_modelx(wtsnm, cfg, device = args.device):
model = TransformerNet(cfg)
 checkpoint = torch.load(wtsnm, map_location=torch.device(device))
model.load_state_dict(checkpoint)
 model = model.half().to(device)
 model = model.eval()
return model
df['isOriginalLanguageEng'] = 0
df.loc[ df['original_language'] == "en" ,"isOriginalLanguageEng"] = 1<categorify>
imgmodels = {}
for fold in args.folds:
wtsnm = f'.. /input/rsnastr2020weights/classifier__RSNAClassifier_tf_efficientnet_b5_ns_04d_fold{fold}_img512_accum1___best'
logger.info(f'Load image model : {wtsnm}')
 imgmodels[fold] = load_image_model(wtsnm, conf = cfgimg, device = args.device)
 exammodels = defaultdict(list)
 embed_size = 2048
for fold in args.folds:
wtsnm = f'.. /input/rsnastr2020weights/exam_lstm_classifier__RSNAClassifier_tf_efficientnet_b5_ns_04d_fold{fold}_img512_accum1___best__all_size512_hidden512_best.bin'
logger.info(f'Load lstm exam model : {wtsnm}')
 exammodels[fold].append(load_exam_model(wtsnm, cfgseq1, embed_size = embed_size, device = args.device))
exammodelsx = defaultdict(list)
 for fold in args.folds:
for cfg in [cfgseq2, cfgseq3]:
wtsnm = '.. /input/rsnastr2020weights/exam_transformer_classifier__RSNAClassifier_tf_efficientnet_b5_ns_04d_'
wtsnm += f'fold{fold}_img512_accum1___best__all_size512_nlayers{cfg.nlayers}_intermediate{cfg.intermediate_size}_hidden{cfg.hidden_size}_best.bin'
logger.info(f'Load xfrmr exam model : {wtsnm}')
 exammodelsx[fold].append(load_exam_modelx(wtsnm, cfg, device = args.device))
dfdic_feature["original_language"] = pd.get_dummies(df["original_language"])
 dfdic_feature["original_language"].head()<categorify>
subdf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/sample_submission.csv')
 subdf = subdf.set_index('id')
 mean_ = torch.tensor(cfgimg.normalize['mean'])[None, None, None, :].to(args.device)
 std_ = torch.tensor(cfgimg.normalize['std'])[None, None, None, :].to(args.device)
 subdf.label = 0.49999
le = LabelEncoder()
df['original_language'] = le.fit_transform(df['original_language'] )<feature_engineering>
tstdf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/test.csv')
 subdf = clean_sub(subdf, tstdf )
df["production_names"] = df["production_companies"].apply(lambda x : [ i["name"] for i in x] )<define_variables>
errors = check_consistency(subdf, tstdf)
 if errors.shape[0] == 0:
subdf.to_csv('submission.csv', index = False)
 else:
print(errors.broken_rule.value_counts() )
tmp = count_word_list(df["production_names"] )<sort_values>
((subdf.label - suborig.label)!=0 ).value_counts()