Kaggle competition: RSNA STR Pulmonary Embolism Detection
<SOS> metric: weightedmeancolumnwiselogloss Kaggle data source: rsna-str-pulmonary-embolism-detection<feature_engineering>
%%capture
!pip install.. /input/rsnapkgs/timm-0.2.2-py3-none-any.whl
!pip install.. /input/pytorch-transformers/pytorch_transformers-1.2.0-py3-none-any.whl
!cp.. /input/gdcm-conda-install/gdcm.tar.
 !tar -xvzf gdcm.tar
!conda install --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2
print("done" )
df['_tagline']=df['tagline'].apply(lambda x : str(x)
 ).apply(lambda x : lower_text(x)).apply(lambda x : remove_punct(x)) 
<feature_engineering>
PATH = '.. /input/rsnasubcheck'
sys.path.append(PATH)
 PATH = '.. /input/rsnastr/rsnastr-master'
sys.path.append(PATH)
 np.set_printoptions(suppress=True)
df['title_word_count'] = df['title'].apply(lambda x: len(str(x ).split())) 
<define_variables>
logger = get_logger('Kaggle', 'INFO' )
df_use2 = df[["has_homepage","runtime",'budget']]<feature_engineering>
conf = load_config(f'{PATH}/configs/512/effnetb5_lr5e4_multi.json')
 class args:
device = 'cuda' if torch.cuda.is_available() else 'cpu'
batchsize = 1
imgbatchsize = 128
folds = [0,1,2]
lstm_units = 512
dropout = 0.0
load_train = False
do_full = True
delta = False
class cfg:
dropout=0.2
hidden_size=2048
intermediate_size=2048
max_position_embeddings=1536
nlayers=1
nheads=8
device=args.device
seed=7
list_of_cast_names = list(df['cast'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 df['num_cast'] = df['cast'].apply(lambda x: len(x)if x != {} else 0)
<feature_engineering>
def create_val_transforms() :
return A.Compose([
A.Normalize(mean=conf['normalize']['mean'],
std=conf['normalize']['std'], max_pixel_value=255.0, p=1.0),
ToTensor()
])
 class RSNAImageSequenceDataset(Dataset):
def __init__(self,
datadf,
folddf,
transforms,
mode="train"):
self.mode = mode
self.transform = transforms
self.datadf = datadf
self.folddf = folddf
self.imgclasses = CFG['image_target_cols']
self.studyclasses = CFG['exam_target_cols']
def __len__(self):
return len(self.folddf)
 def __getitem__(self, idx):
studyidx = self.folddf.iloc[idx].StudyInstanceUID
seriesidx = self.folddf.iloc[idx].SeriesInstanceUID
studydf = self.datadf.query('StudyInstanceUID == @studyidx')\ 
.query('SeriesInstanceUID == @seriesidx')
 imgs = []
imgnames = []
imglabels = []
if self.mode == 'train':
studylabels = studydf[self.studyclasses].iloc[0].values
for i, samp in studydf.reset_index().iterrows() :
if self.mode == 'train':
imglabels.append(samp['pe_present_on_image'])
 sampkey, img_name = self.image_file(samp)
 try:
dicom_object = pydicom.read_file(img_name)
 img = ip_window(dicom_object)
 except:
img = np.zeros(shape=(512,512,3), dtype=np.uint8)

imgs.append(img)
 imgnames.append(sampkey)
 imgs = np.stack(imgs)
 if self.mode == 'train':
labels = torch.tensor(imglabels)
 return {'img_name': imgnames, 'image': imgs, 'imglabels': imglabels, 'studylabels': studylabels}
return {'img_name': imgnames, 'image': imgs}
def image_file(self, samp):
sampkey = '/'.join([samp.StudyInstanceUID, samp.SeriesInstanceUID,samp.SOPInstanceUID])
 imgname = os.path.join('.. /input/rsna-str-pulmonary-embolism-detection',
self.mode,
sampkey)+ '.dcm'
return sampkey, imgname
def collateimgfn(batch):
maxlen = max([l['image'].shape[0] for l in batch])
 for b in batch:
masklen = maxlen-len(b['image'])
 b['mask'] = np.ones(( maxlen))
b['mask'][:masklen] = 0
outbatch = {'image': np.concatenate([b['image'] for b in batch], 0)}
outbatch['mask'] = torch.tensor(np.vstack([np.expand_dims(b['mask'], 0)\
for b in batch])).float()
outbatch['img_name'] = []
for b in batch:
outbatch['img_name'] += b['img_name']
if 'imglabels' in batch[0]:
for b in batch:
masklen = maxlen-len(b['image'])
 b['imglabelsout'] = np.ones(( maxlen)) *-1
b['imglabelsout'][masklen:] = b['imglabels']
outbatch['imglabels'] = torch.tensor(np.vstack([np.expand_dims(b['imglabelsout'], 0)\
for b in batch])).long()
outbatch['studylabels'] = torch.tensor(np.vstack([np.expand_dims(b['studylabels'], 0)\
for b in batch])).long()
return outbatch
class TransformerNet(nn.Module):
def __init__(self,
cfg,
nimgclasses = 1,
nstudyclasses = 9):
super(TransformerNet, self ).__init__()
self.nimgclasses = nimgclasses
self.nstudyclasses = nstudyclasses
self.cfg = cfg
self.config = BertConfig(
 3,
hidden_size=cfg.hidden_size,
num_hidden_layers=cfg.nlayers,
num_attention_heads=cfg.nheads,
max_position_embeddings=cfg.max_position_embeddings,
intermediate_size=cfg.intermediate_size,
hidden_dropout_prob=cfg.dropout,
attention_probs_dropout_prob=cfg.dropout, 
)
 self.encoder = BertEncoder(self.config ).to(cfg.device)
 self.img_linear_out = nn.Linear(cfg.hidden_size, self.nimgclasses)
 self.study_linear_out = nn.Linear(cfg.hidden_size, self.nstudyclasses)
 def extended_mask(self, mask):
extended_attention_mask = mask.unsqueeze(1 ).unsqueeze(2)
 extended_attention_mask =(1.0 - extended_attention_mask)* -10000.0
head_mask = [None] * self.cfg.nlayers
return extended_attention_mask, head_mask
def forward(self, x, mask, lengths=None):
extended_attention_mask, head_mask = extended_mask(mask)
 encoded_layers = encoder(x, extended_attention_mask, head_mask=head_mask)
 sequence_output = encoded_layers[-1]
img_output = self.img_linear_out(sequence_output ).squeeze()
study_output = self.study_linear_out(sequence_output[:, -1] ).squeeze()
return study_output, img_output
def batchPredsfn(studypreds, imgpreds, imgnames):
preds = []
batchdf = pd.DataFrame([i.split('/')for i in imgnames],
columns = ['StudyInstanceUID', 'SeriesInstanceUID','SOPInstanceUID'])
 studies = batchdf.StudyInstanceUID.unique().tolist()
for study, row in zip(studies, studypreds):
for col, pred in zip(CFG['exam_target_cols'], row):
preds.append([f'{study}_{col}', pred])
 for nm, pred in zip(batchdf.SOPInstanceUID, imgpreds):
preds.append([nm, pred])
 return preds
def aug_batch(imgs, augfn = create_val_transforms()):
imgbatch = []
for i in imgs:
augmented = augfn(image=i)
 i = augmented['image'].half()
imgbatch.append(i)
 imgbatch = torch.stack(imgbatch)
 return imgbatch
def aug_batch1(imgs, mean_, std_, device):
imgs = imgs.to(device ).float()
return(((imgs / 255)- mean_)/ std_ ).permute(0, 3, 1, 2 ).half()
list_of_cast_genders = list(df['cast'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 df['genders_0_cast'] = df['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
df['genders_1_cast'] = df['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
df['genders_2_cast'] = df['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))<feature_engineering>
!cp.. /input/rsna-str-pulmonary-embolism-detection/sample_submission.csv submission.csv
if args.load_train:
trndf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/train.csv')
 if os.path.exists('.. /input/rsna-str-pulmonary-embolism-detection/train')and not args.do_full:
tstdf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/test.csv' ).head(2000)
 else:
tstdf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/test.csv')
 keycols = ['StudyInstanceUID', 'SeriesInstanceUID']
DEBUG =(tstdf.shape[0]==146853)
 DEBUG
list_of_crew_names = list(df['crew'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 df['num_crew'] = df['crew'].apply(lambda x: len(x)if x != {} else 0)
<create_dataframe>
loaderargs = {'num_workers' : 4, 'pin_memory': False, 'drop_last': False, 'collate_fn' : collateimgfn}
if args.load_train:
trndataset = RSNAImageSequenceDataset(mode = 'train',
folddf = trndf[keycols].drop_duplicates().reset_index(drop=True),
datadf = trndf,
transforms = create_val_transforms())
 trnloader = DataLoader(trndataset, batch_size=args.batchsize, shuffle=False, **loaderargs)
 tstdataset = RSNAImageSequenceDataset(mode = 'test',
folddf = tstdf[keycols].drop_duplicates().reset_index(drop=True),
datadf = tstdf,
transforms = create_val_transforms())
 tstloader = DataLoader(tstdataset, batch_size=args.batchsize, shuffle=False, **loaderargs )
df_crew = { idx : pd.DataFrame([ [crew["department"], crew["job"], crew["name"]]
for crew in x], columns=["department", "job", "name"])
 for idx, x in df["crew"].iteritems() }<concatenate>
def load_image_model(wtsnm, device = args.device):
nclasses = len(conf["image_target_cols"])+ len(conf['exam_target_cols'])
 model = classifiers.__dict__[conf['network']](encoder=f"{conf['encoder']}_infer",
nclasses = nclasses,
infer = True)
 checkpoint = torch.load(wtsnm, map_location=torch.device(device))
model.load_state_dict(checkpoint['state_dict'])
 model = model.half().to(device)
 model = model.eval()
return model
def load_exam_model(wtsnm, embed_size = 2048, device = args.device):
model = LSTMNet(embed_size,
nimgclasses = len(CFG["image_target_cols"]),
nstudyclasses = len(CFG['exam_target_cols']),
LSTM_UNITS=args.lstm_units,
DO = args.dropout)
 checkpoint = torch.load(wtsnm, map_location=torch.device(device))
model.load_state_dict(checkpoint)
 model = model.half().to(device)
 model = model.eval()
return model
def load_exam_modelx(wtsnm, cfg, device = args.device):
model = TransformerNet(cfg)
 checkpoint = torch.load(wtsnm, map_location=torch.device(device))
model.load_state_dict(checkpoint)
 model = model.half().to(device)
 model = model.eval()
return model
df_crew = pd.concat(df_crew )<feature_engineering>
imgmodels = {}
for fold in args.folds:
wtsnm = f'.. /input/rsna512effnetb5/classifier_RSNAClassifier_tf_efficientnet_b5_ns_04d_{fold}__nclasses1_fold{fold}_epoch20'
wtsnm = f'.. /input/rsna512effnetb5examwts/classifier_RSNAClassifier_tf_efficientnet_b5_ns_04d_{fold}__nclasses10_size512_fold{fold}_epoch20'
logger.info(f'Load image model : {wtsnm}')
 imgmodels[fold] = load_image_model(wtsnm, device = args.device )
def select_job(list_dict, key, value):
return [ dic["name"] for dic in list_dict if dic[key]==value]<feature_engineering>
exammodels = defaultdict(list)
 embed_size = 2048
if args.delta:
embed_size *= 3
for fold in args.folds:
for epoch in [12]:
wtsnm = '.. /input/rsna512lstmb5examwts/exam_lstm_classifier_RSNAClassifier_tf_efficientnet'
wtsnm += f'_b5_ns_04d_{fold}__nclasses10_size512_fold{fold}_epoch20__all_size512__epoch{epoch}.bin'
logger.info(f'Load exam model : {wtsnm}')
 exammodels[fold].append(load_exam_model(wtsnm, embed_size = embed_size, device = args.device))
for department in department_count.index:
df['dep_{}_num'.format(department)] = df["crew"].apply(select_job, key="department", value=department ).apply(len )<feature_engineering>
exammodelsx = defaultdict(list)
 for fold in args.folds:
for epoch in [12]:
wtsnm = '.. /input/rsna512lstmb5examwts/exam_transformer_classifier_RSNAClassifier_tf_efficientnet'
wtsnm += f'_b5_ns_04d_{fold}__nclasses10_size512_fold{fold}_epoch20__all_size512_nlayers1_hidden2048__epoch{epoch}.bin'
logger.info(f'Load exam model : {wtsnm}')
 exammodelsx[fold].append(load_exam_modelx(wtsnm, cfg, device = args.device))
df_crewname = pd.DataFrame([], index=df.index)
 for job in ["Producer", "Director", "Screenplay", "Casting", "Original Music Composer"]:
col = 'job_{}_list'.format(job)
 df[col] = df["crew"].apply(select_job, key="job", value=job)
 top_list = [m[0] for m in Counter([i for j in df[col] for i in j] ).most_common(15)]
for i in top_list:
df_crewname['{}_{}'.format(job,i)] = df[col].apply(lambda x: i in x )<feature_engineering>
subdf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/sample_submission.csv')
 subdf = subdf.set_index('id')
 pbar = tqdm(enumerate(tstloader), total = 1 + len(tstdataset)//tstloader.batch_size, ncols=0)
 mean_ = torch.tensor(conf['normalize']['mean'])[None, None, None, :].to(args.device)
 std_ = torch.tensor(conf['normalize']['std'])[None, None, None, :].to(args.device)
 for step, batch in pbar:
try:
emb = defaultdict(list)
 studypredsls = []
imgpredsls = []
embbatch = {}
imgs = torch.tensor(batch["image"])
 imgnames = batch['img_name']
mask = batch['mask'].half().to(args.device)
 for imgchunk in torch.split(imgs, args.imgbatchsize, dim=0):
imgchunk = aug_batch1(imgchunk, mean_, std_, args.device)
 imgchunk = imgchunk.to(args.device)
 with torch.no_grad() :
for fold in args.folds:
emb[fold].append(imgmodels[fold](imgchunk))
del imgchunk, imgs, batch
for fold in args.folds:
emb[fold] = torch.cat(emb[fold], 0)
 embbatch[fold] = torch.zeros(len(mask.flatten()), emb[fold].shape[-1] ).half().to(args.device)
 embbatch[fold][mask.flatten() ==1] = emb[fold]
embbatch[fold] = embbatch[fold].reshape(( *mask.shape, emb[fold].shape[-1]))
del emb[fold]
for mod in exammodels[fold]:
with torch.no_grad() :
studylogits, imglogits = mod(embbatch[fold], mask)
 imglogits = imglogits.flatten() [(mask==1 ).flatten() ]
studypredsls.append(torch.sigmoid(studylogits ).detach().cpu().numpy())
 imgpredsls.append(torch.sigmoid(imglogits ).detach().cpu().numpy())
 try:
for mod in exammodelsx[fold]:
with torch.no_grad() :
encoded_layers = mod.encoder(embbatch[fold], *mod.extended_mask(mask))
imglogits = mod.img_linear_out(encoded_layers[-1] ).squeeze()
studylogits = mod.study_linear_out(encoded_layers[-1][:, -1] ).squeeze()
imglogits = imglogits.flatten() [(mask==1 ).flatten() ]
studypredsls.append(torch.sigmoid(studylogits ).detach().cpu().numpy())
 imgpredsls.append(torch.sigmoid(imglogits ).detach().cpu().numpy())
 except:
logger.info(f'Transformer failed...{e}')
 studypreds = sum(studypredsls)/len(studypredsls)
 imgpreds = sum(imgpredsls)/len(imgpredsls)
 subbatch = batchPredsfn(studypreds, imgpreds, imgnames)
 subbatch = pd.DataFrame(subbatch, columns = ['id', 'label'])
 subdf.loc[subbatch.id, 'label'] = subbatch.label.values
del mask, embbatch, imglogits, studylogits
except Exception as e:
logger.info(f'Failed...{e}')
 if step % 1 == 0:
torch.cuda.empty_cache()
if step % 5 == 0:
gc.collect()
subdf = subdf.reset_index()
suborig = subdf.copy()
for job in ["Sound", "Art", "Costume & Make-Up", "Camera", "Visual Effects"]:
col = 'department_{}_list'.format(job)
 df[col] = df["crew"].apply(select_job, key="department", value=job)
 top_list = [m[0] for m in Counter([i for j in df[col] for i in j] ).most_common(15)]
for i in top_list:
df_crewname['{}_{}'.format(job,i)] = df[col].apply(lambda x: i in x )<load_pretrained>
tstdf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/test.csv')
 subdf = clean_sub(subdf, tstdf )
with open("/kaggle/input/private-jhk/df_use_nagano.pkl","rb")as fr:
df_use_nagano = pickle.load(fr )<drop_column>
errors = check_consistency(subdf, tstdf)
 if errors.shape[0] == 0:
subdf.to_csv('submission.csv', index = False)
 else:
print(errors.broken_rule.value_counts() )
<filter><EOS> 
((subdf.label - suborig.label)!=0 ).value_counts()