Kaggle competition: Titanic - Machine Learning from Disaster
%%capture
!pip install.. /input/rsnapkgs/timm-0.2.2-py3-none-any.whl
!pip install.. /input/pytorch-transformers/pytorch_transformers-1.2.0-py3-none-any.whl
!cp.. /input/gdcm-conda-install/gdcm.tar.
 !tar -xvzf gdcm.tar
!conda install --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2
print("done" )<import_modules>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
PATH = '.. /input/rsnasubcheck'
sys.path.append(PATH)
 PATH = '.. /input/rsnastr/rsnastr-master'
sys.path.append(PATH)
 np.set_printoptions(suppress=True)
<import_modules>
train_x = train.drop(['Survived'], axis=1)
 train_y = train['Survived']
logger = get_logger('Kaggle', 'INFO' )<init_hyperparams>
test_x = test.copy()
conf = load_config(f'{PATH}/configs/512/effnetb5_lr5e4_multi.json')
 class args:
device = 'cuda' if torch.cuda.is_available() else 'cpu'
batchsize = 1
imgbatchsize = 128
folds = [0,1,2]
lstm_units = 512
dropout = 0.0
load_train = False
do_full = True
delta = False
class cfg:
dropout=0.2
hidden_size=2048
intermediate_size=2048
max_position_embeddings=1536
nlayers=1
nheads=8
device=args.device
seed=7<categorify>
from sklearn.preprocessing import LabelEncoder
def create_val_transforms() :
return A.Compose([
A.Normalize(mean=conf['normalize']['mean'],
std=conf['normalize']['std'], max_pixel_value=255.0, p=1.0),
ToTensor()
])
 class RSNAImageSequenceDataset(Dataset):
def __init__(self,
datadf,
folddf,
transforms,
mode="train"):
self.mode = mode
self.transform = transforms
self.datadf = datadf
self.folddf = folddf
self.imgclasses = CFG['image_target_cols']
self.studyclasses = CFG['exam_target_cols']
def __len__(self):
return len(self.folddf)
 def __getitem__(self, idx):
studyidx = self.folddf.iloc[idx].StudyInstanceUID
seriesidx = self.folddf.iloc[idx].SeriesInstanceUID
studydf = self.datadf.query('StudyInstanceUID == @studyidx')\ 
.query('SeriesInstanceUID == @seriesidx')
 imgs = []
imgnames = []
imglabels = []
if self.mode == 'train':
studylabels = studydf[self.studyclasses].iloc[0].values
for i, samp in studydf.reset_index().iterrows() :
if self.mode == 'train':
imglabels.append(samp['pe_present_on_image'])
 sampkey, img_name = self.image_file(samp)
 try:
dicom_object = pydicom.read_file(img_name)
 img = ip_window(dicom_object)
 except:
img = np.zeros(shape=(512,512,3), dtype=np.uint8)

imgs.append(img)
 imgnames.append(sampkey)
 imgs = np.stack(imgs)
 if self.mode == 'train':
labels = torch.tensor(imglabels)
 return {'img_name': imgnames, 'image': imgs, 'imglabels': imglabels, 'studylabels': studylabels}
return {'img_name': imgnames, 'image': imgs}
def image_file(self, samp):
sampkey = '/'.join([samp.StudyInstanceUID, samp.SeriesInstanceUID,samp.SOPInstanceUID])
 imgname = os.path.join('.. /input/rsna-str-pulmonary-embolism-detection',
self.mode,
sampkey)+ '.dcm'
return sampkey, imgname
def collateimgfn(batch):
maxlen = max([l['image'].shape[0] for l in batch])
 for b in batch:
masklen = maxlen-len(b['image'])
 b['mask'] = np.ones(( maxlen))
b['mask'][:masklen] = 0
outbatch = {'image': np.concatenate([b['image'] for b in batch], 0)}
outbatch['mask'] = torch.tensor(np.vstack([np.expand_dims(b['mask'], 0)\
for b in batch])).float()
outbatch['img_name'] = []
for b in batch:
outbatch['img_name'] += b['img_name']
if 'imglabels' in batch[0]:
for b in batch:
masklen = maxlen-len(b['image'])
 b['imglabelsout'] = np.ones(( maxlen)) *-1
b['imglabelsout'][masklen:] = b['imglabels']
outbatch['imglabels'] = torch.tensor(np.vstack([np.expand_dims(b['imglabelsout'], 0)\
for b in batch])).long()
outbatch['studylabels'] = torch.tensor(np.vstack([np.expand_dims(b['studylabels'], 0)\
for b in batch])).long()
return outbatch
class TransformerNet(nn.Module):
def __init__(self,
cfg,
nimgclasses = 1,
nstudyclasses = 9):
super(TransformerNet, self ).__init__()
self.nimgclasses = nimgclasses
self.nstudyclasses = nstudyclasses
self.cfg = cfg
self.config = BertConfig(
 3,
hidden_size=cfg.hidden_size,
num_hidden_layers=cfg.nlayers,
num_attention_heads=cfg.nheads,
max_position_embeddings=cfg.max_position_embeddings,
intermediate_size=cfg.intermediate_size,
hidden_dropout_prob=cfg.dropout,
attention_probs_dropout_prob=cfg.dropout, 
)
 self.encoder = BertEncoder(self.config ).to(cfg.device)
 self.img_linear_out = nn.Linear(cfg.hidden_size, self.nimgclasses)
 self.study_linear_out = nn.Linear(cfg.hidden_size, self.nstudyclasses)
 def extended_mask(self, mask):
extended_attention_mask = mask.unsqueeze(1 ).unsqueeze(2)
 extended_attention_mask =(1.0 - extended_attention_mask)* -10000.0
head_mask = [None] * self.cfg.nlayers
return extended_attention_mask, head_mask
def forward(self, x, mask, lengths=None):
extended_attention_mask, head_mask = extended_mask(mask)
 encoded_layers = encoder(x, extended_attention_mask, head_mask=head_mask)
 sequence_output = encoded_layers[-1]
img_output = self.img_linear_out(sequence_output ).squeeze()
study_output = self.study_linear_out(sequence_output[:, -1] ).squeeze()
return study_output, img_output
def batchPredsfn(studypreds, imgpreds, imgnames):
preds = []
batchdf = pd.DataFrame([i.split('/')for i in imgnames],
columns = ['StudyInstanceUID', 'SeriesInstanceUID','SOPInstanceUID'])
 studies = batchdf.StudyInstanceUID.unique().tolist()
for study, row in zip(studies, studypreds):
for col, pred in zip(CFG['exam_target_cols'], row):
preds.append([f'{study}_{col}', pred])
 for nm, pred in zip(batchdf.SOPInstanceUID, imgpreds):
preds.append([nm, pred])
 return preds
def aug_batch(imgs, augfn = create_val_transforms()):
imgbatch = []
for i in imgs:
augmented = augfn(image=i)
 i = augmented['image'].half()
imgbatch.append(i)
 imgbatch = torch.stack(imgbatch)
 return imgbatch
def aug_batch1(imgs, mean_, std_, device):
imgs = imgs.to(device ).float()
return(((imgs / 255)- mean_)/ std_ ).permute(0, 3, 1, 2 ).half()<load_from_csv>
train_x = train_x.drop(['PassengerId'], axis=1)
 test_x = test_x.drop(['PassengerId'], axis=1 )
!cp.. /input/rsna-str-pulmonary-embolism-detection/sample_submission.csv submission.csv
if args.load_train:
trndf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/train.csv')
 if os.path.exists('.. /input/rsna-str-pulmonary-embolism-detection/train')and not args.do_full:
tstdf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/test.csv' ).head(2000)
 else:
tstdf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/test.csv')
 keycols = ['StudyInstanceUID', 'SeriesInstanceUID']
DEBUG =(tstdf.shape[0]==146853)
 DEBUG<create_dataframe>
train_x = train_x.drop(['Name', 'Ticket', 'Cabin'], axis=1)
 test_x = test_x.drop(['Name', 'Ticket', 'Cabin'], axis=1 )
loaderargs = {'num_workers' : 4, 'pin_memory': False, 'drop_last': False, 'collate_fn' : collateimgfn}
if args.load_train:
trndataset = RSNAImageSequenceDataset(mode = 'train',
folddf = trndf[keycols].drop_duplicates().reset_index(drop=True),
datadf = trndf,
transforms = create_val_transforms())
 trnloader = DataLoader(trndataset, batch_size=args.batchsize, shuffle=False, **loaderargs)
 tstdataset = RSNAImageSequenceDataset(mode = 'test',
folddf = tstdf[keycols].drop_duplicates().reset_index(drop=True),
datadf = tstdf,
transforms = create_val_transforms())
 tstloader = DataLoader(tstdataset, batch_size=args.batchsize, shuffle=False, **loaderargs )<load_pretrained>
for c in ['Sex', 'Embarked']:
le = LabelEncoder()
le.fit(train_x[c].fillna('NA'))
train_x[c] = le.transform(train_x[c].fillna('NA'))
test_x[c] = le.transform(test_x[c].fillna('NA'))
def load_image_model(wtsnm, device = args.device):
nclasses = len(conf["image_target_cols"])+ len(conf['exam_target_cols'])
 model = classifiers.__dict__[conf['network']](encoder=f"{conf['encoder']}_infer",
nclasses = nclasses,
infer = True)
 checkpoint = torch.load(wtsnm, map_location=torch.device(device))
model.load_state_dict(checkpoint['state_dict'])
 model = model.half().to(device)
 model = model.eval()
return model
def load_exam_model(wtsnm, embed_size = 2048, device = args.device):
model = LSTMNet(embed_size,
nimgclasses = len(CFG["image_target_cols"]),
nstudyclasses = len(CFG['exam_target_cols']),
LSTM_UNITS=args.lstm_units,
DO = args.dropout)
 checkpoint = torch.load(wtsnm, map_location=torch.device(device))
model.load_state_dict(checkpoint)
 model = model.half().to(device)
 model = model.eval()
return model
def load_exam_modelx(wtsnm, cfg, device = args.device):
model = TransformerNet(cfg)
 checkpoint = torch.load(wtsnm, map_location=torch.device(device))
model.load_state_dict(checkpoint)
 model = model.half().to(device)
 model = model.eval()
return model<load_pretrained>
from xgboost import XGBClassifier
imgmodels = {}
for fold in args.folds:
wtsnm = f'.. /input/rsna512effnetb5/classifier_RSNAClassifier_tf_efficientnet_b5_ns_04d_{fold}__nclasses1_fold{fold}_epoch20'
wtsnm = f'.. /input/rsna512effnetb5examwts/classifier_RSNAClassifier_tf_efficientnet_b5_ns_04d_{fold}__nclasses10_size512_fold{fold}_epoch20'
logger.info(f'Load image model : {wtsnm}')
 imgmodels[fold] = load_image_model(wtsnm, device = args.device )<define_variables>
model = XGBClassifier(n_estimators=20, random_state=71)
 model.fit(train_x, train_y )
exammodels = defaultdict(list)
 embed_size = 2048
if args.delta:
embed_size *= 3
for fold in args.folds:
for epoch in [12]:
wtsnm = '.. /input/rsna512lstmb5examwts/exam_lstm_classifier_RSNAClassifier_tf_efficientnet'
wtsnm += f'_b5_ns_04d_{fold}__nclasses10_size512_fold{fold}_epoch20__all_size512__epoch{epoch}.bin'
logger.info(f'Load exam model : {wtsnm}')
 exammodels[fold].append(load_exam_model(wtsnm, embed_size = embed_size, device = args.device))<define_variables>
pred = model.predict_proba(test_x)[:, 1]
exammodelsx = defaultdict(list)
 for fold in args.folds:
for epoch in [12]:
wtsnm = '.. /input/rsna512lstmb5examwts/exam_transformer_classifier_RSNAClassifier_tf_efficientnet'
wtsnm += f'_b5_ns_04d_{fold}__nclasses10_size512_fold{fold}_epoch20__all_size512_nlayers1_hidden2048__epoch{epoch}.bin'
logger.info(f'Load exam model : {wtsnm}')
 exammodelsx[fold].append(load_exam_modelx(wtsnm, cfg, device = args.device)) 
<load_from_csv>
pred_label = np.where(pred > 0.5, 1, 0 )
subdf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/sample_submission.csv')
 subdf = subdf.set_index('id')
 pbar = tqdm(enumerate(tstloader), total = 1 + len(tstdataset)//tstloader.batch_size, ncols=0)
 mean_ = torch.tensor(conf['normalize']['mean'])[None, None, None, :].to(args.device)
 std_ = torch.tensor(conf['normalize']['std'])[None, None, None, :].to(args.device)
 for step, batch in pbar:
try:
emb = defaultdict(list)
 studypredsls = []
imgpredsls = []
embbatch = {}
imgs = torch.tensor(batch["image"])
 imgnames = batch['img_name']
mask = batch['mask'].half().to(args.device)
 for imgchunk in torch.split(imgs, args.imgbatchsize, dim=0):
imgchunk = aug_batch1(imgchunk, mean_, std_, args.device)
 imgchunk = imgchunk.to(args.device)
 with torch.no_grad() :
for fold in args.folds:
emb[fold].append(imgmodels[fold](imgchunk))
del imgchunk, imgs, batch
for fold in args.folds:
emb[fold] = torch.cat(emb[fold], 0)
 embbatch[fold] = torch.zeros(len(mask.flatten()), emb[fold].shape[-1] ).half().to(args.device)
 embbatch[fold][mask.flatten() ==1] = emb[fold]
embbatch[fold] = embbatch[fold].reshape(( *mask.shape, emb[fold].shape[-1]))
del emb[fold]
for mod in exammodels[fold]:
with torch.no_grad() :
studylogits, imglogits = mod(embbatch[fold], mask)
 imglogits = imglogits.flatten() [(mask==1 ).flatten() ]
studypredsls.append(torch.sigmoid(studylogits ).detach().cpu().numpy())
 imgpredsls.append(torch.sigmoid(imglogits ).detach().cpu().numpy())
 try:
for mod in exammodelsx[fold]:
with torch.no_grad() :
encoded_layers = mod.encoder(embbatch[fold], *mod.extended_mask(mask))
imglogits = mod.img_linear_out(encoded_layers[-1] ).squeeze()
studylogits = mod.study_linear_out(encoded_layers[-1][:, -1] ).squeeze()
imglogits = imglogits.flatten() [(mask==1 ).flatten() ]
studypredsls.append(torch.sigmoid(studylogits ).detach().cpu().numpy())
 imgpredsls.append(torch.sigmoid(imglogits ).detach().cpu().numpy())
 except:
logger.info(f'Transformer failed...{e}')
 studypreds = sum(studypredsls)/len(studypredsls)
 imgpreds = sum(imgpredsls)/len(imgpredsls)
 subbatch = batchPredsfn(studypreds, imgpreds, imgnames)
 subbatch = pd.DataFrame(subbatch, columns = ['id', 'label'])
 subdf.loc[subbatch.id, 'label'] = subbatch.label.values
del mask, embbatch, imglogits, studylogits
except Exception as e:
logger.info(f'Failed...{e}')
 if step % 1 == 0:
torch.cuda.empty_cache()
if step % 5 == 0:
gc.collect()
subdf = subdf.reset_index()
suborig = subdf.copy()<load_from_csv>
submission = pd.DataFrame({'PassengerId': test['PassengerId'], 'Survived': pred_label})
 submission.to_csv('submission_first.csv', index=False )
tstdf = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/test.csv')
 subdf = clean_sub(subdf, tstdf )<save_to_csv>
from sklearn.metrics import log_loss, accuracy_score
from sklearn.model_selection import KFold
errors = check_consistency(subdf, tstdf)
 if errors.shape[0] == 0:
subdf.to_csv('submission.csv', index = False)
 else:
print(errors.broken_rule.value_counts() )<count_values>
scores_accuracy = []
scores_logloss = []
((subdf.label - suborig.label)!=0 ).value_counts()<install_modules>
kf = KFold(n_splits=4, shuffle=True, random_state=71)
 for tr_idx, va_idx in kf.split(train_x):
tr_x, va_x = train_x.iloc[tr_idx], train_x.iloc[va_idx]
tr_y, va_y = train_y.iloc[tr_idx], train_y.iloc[va_idx]
model = XGBClassifier(n_estimators=20, random_state=71)
 model.fit(tr_x, tr_y)
 va_pred = model.predict_proba(va_x)[:, 1]
logloss = log_loss(va_y, va_pred)
 accuracy = accuracy_score(va_y, va_pred > 0.5)
 scores_logloss.append(logloss)
 scores_accuracy.append(accuracy )
package_path = '.. /input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'
!cp.. /input/gdcm-conda-install/gdcm.tar.
 !tar -xvzf gdcm.tar
!conda install --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2 
<define_variables>
logloss = np.mean(scores_logloss)
 accuracy = np.mean(scores_accuracy)
 print(f'logloss: {logloss:.4f}, accuracy: {accuracy:.4f}' )
CFG = {
'train': False,
'train_img_path': '.. /input/rsna-str-pulmonary-embolism-detection/train',
'test_img_path': '.. /input/rsna-str-pulmonary-embolism-detection/test',
'cv_fold_path': '.. /input/stratified-validation-strategy/rsna_train_splits_fold_20.csv',
'train_path': '.. /input/rsna-str-pulmonary-embolism-detection/train.csv',
'test_path': '.. /input/rsna-str-pulmonary-embolism-detection/test.csv',
'image_target_cols': [
'pe_present_on_image',
],
'exam_target_cols': [
'negative_exam_for_pe',
'rv_lv_ratio_gte_1',
'rv_lv_ratio_lt_1',
'leftsided_pe',
'chronic_pe',
'rightsided_pe',
'acute_and_chronic_pe',
'central_pe',
'indeterminate'
],
'img_num': 200,
'img_size': 256,
'lr': 0.0005,
'epochs': 2,
'device': 'cuda',
'train_bs': 2,
'accum_iter': 8,
'verbose_step': 1,
'num_workers': 4,
'efbnet': 'efficientnet-b0',
'train_folds': [np.arange(0,16),
np.concatenate([np.arange(0,12), np.arange(16,20)]),
np.concatenate([np.arange(0,8), np.arange(12,20)]),
np.concatenate([np.arange(0,4), np.arange(8,20)]),
np.arange(4,20),
],
'valid_folds': [np.arange(16,20),
np.arange(12,16),
np.arange(8,12),
np.arange(4,8),
np.arange(0,4)
 ],
'model_path': '.. /input/kh-rsna-model',
'tag': 'efb0_stage2_multilabel'
}
SEED = 42321
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def window(img, WL=50, WW=350):
upper, lower = WL+WW//2, WL-WW//2
X = np.clip(img.copy() , lower, upper)
 X = X - np.min(X)
 X = X / np.max(X)
 return X
def get_img(path):
d = pydicom.read_file(path)


 img =(d.pixel_array * d.RescaleSlope)+ d.RescaleIntercept
r = window(img, -600, 1500)
 g = window(img, 100, 700)
 b = window(img, 40, 400)
 res = np.concatenate([r[:, :, np.newaxis],
g[:, :, np.newaxis],
b[:, :, np.newaxis]], axis=-1)
 res = zoom(res, [CFG['img_size']/res.shape[0], CFG['img_size']/res.shape[1], 1.], prefilter=False, order=1)
 return res
class RSNADatasetStage1(Dataset):
def __init__(
 self, df, label_smoothing, data_root,
image_subsampling=True, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.label_smoothing = label_smoothing
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df[CFG['image_target_cols']].values[index]
path = "{}/{}/{}/{}.dcm".format(self.data_root,
self.df.iloc[index]['StudyInstanceUID'],
self.df.iloc[index]['SeriesInstanceUID'],
self.df.iloc[index]['SOPInstanceUID'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
target = np.clip(target, self.label_smoothing, 1 - self.label_smoothing)
 return img, target
else:
return img
class RSNADataset(Dataset):
def __init__(
 self, df, label_smoothing, data_root,
image_subsampling=True, transforms=None, output_label=True 
):
super().__init__()
self.df = df
self.patients = self.df['StudyInstanceUID'].unique()
self.image_subsampling = image_subsampling
self.label_smoothing = label_smoothing
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def get_patients(self):
return self.patients
def __len__(self):
return len(self.patients)
 def __getitem__(self, index: int):
patient = self.patients[index]
df_ = self.df.loc[self.df.StudyInstanceUID == patient]
per_image_feats = get_stage1_columns()
if self.image_subsampling:
img_num = min(CFG['img_num'], df_.shape[0])
 img_ix = np.random.choice(np.arange(df_.shape[0]), replace=False, size=img_num)
 imgs = np.zeros(( CFG['img_num'],), np.float32)
 per_image_preds = np.zeros(( CFG['img_num'], len(per_image_feats)) , np.float32)
 locs = np.zeros(( CFG['img_num'],), np.float32)
 image_masks = np.zeros(( CFG['img_num'],), np.float32)
 image_masks[:img_num] = 1.
 if self.output_label:
exam_label = df_[CFG['exam_target_cols']].values[0]
image_labels = np.zeros(( CFG['img_num'], len(CFG['image_target_cols'])) , np.float32)
 else:
img_num = df_.shape[0]
img_ix = np.arange(df_.shape[0])
 imgs = np.zeros(( img_num,), np.float32)
 per_image_preds = np.zeros(( img_num, len(per_image_feats)) , np.float32)
 locs = np.zeros(( img_num,), np.float32)
 image_masks = np.zeros(( img_num,), np.float32)
 image_masks[:img_num] = 1.
 if self.output_label:
exam_label = df_[CFG['exam_target_cols']].values[0]
image_labels = np.zeros(( img_num, len(CFG['image_target_cols'])) , np.float32)
 for i, im_ix in enumerate(img_ix):
path = "{}/{}/{}/{}.dcm".format(self.data_root,
df_['StudyInstanceUID'].values[im_ix],
df_['SeriesInstanceUID'].values[im_ix],
df_['SOPInstanceUID'].values[im_ix])
 d = pydicom.read_file(path)
 locs[i] = d.ImagePositionPatient[2]
per_image_preds[i,:] = df_[per_image_feats].values[im_ix,:]
if self.output_label == True:
image_labels[i] = df_[CFG['image_target_cols']].values[im_ix]
seq_ix = np.argsort(locs)

locs = locs[seq_ix]
locs[1:img_num] = locs[1:img_num]-locs[0:img_num-1]
locs[0] = 0
per_image_preds = per_image_preds[seq_ix]
if self.output_label == True:
image_labels = image_labels[seq_ix]
image_labels = np.clip(image_labels, self.label_smoothing, 1 - self.label_smoothing)
 exam_label = np.clip(exam_label, self.label_smoothing, 1 - self.label_smoothing)
 return imgs, per_image_preds, locs, image_labels, exam_label, image_masks
else:
return imgs, per_image_preds, locs, img_num, index, seq_ix
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout 
)
 def get_train_transforms() :
return Compose([
ToTensorV2(p=1.0),
], p=1.)

 def get_valid_transforms() :
return Compose([
HorizontalFlip(p=0.25),
VerticalFlip(p=0.25),
RandomRotate90(p=0.25),
ToTensorV2(p=1.0),
], p=1.)

 class RNSAImageFeatureExtractor(nn.Module):
def __init__(self):
super().__init__()
self.cnn_model = EfficientNet.from_name(CFG['efbnet'])
 self.pooling = nn.AdaptiveAvgPool2d(1)
 def get_dim(self):
return self.cnn_model._fc.in_features
def forward(self, x):
feats = self.cnn_model.extract_features(x)
 return self.pooling(feats ).view(x.shape[0], -1)
 class RSNAImgClassifierSingle(nn.Module):
def __init__(self):
super().__init__()
self.cnn_model = RNSAImageFeatureExtractor()
self.image_predictors = nn.Linear(self.cnn_model.get_dim() , 1)
 def forward(self, imgs):
imgs_embdes = self.cnn_model(imgs)
 image_preds = self.image_predictors(imgs_embdes)
 return image_preds
class RSNAImgClassifier(nn.Module):
def __init__(self):
super().__init__()
self.cnn_model = RNSAImageFeatureExtractor()
self.image_predictors = nn.Linear(self.cnn_model.get_dim() , 9)
 def forward(self, imgs):
imgs_embdes = self.cnn_model(imgs)
 image_preds = self.image_predictors(imgs_embdes)
 return image_preds
class TimeDistributed(nn.Module):
def __init__(self, module, batch_first=True):
super(TimeDistributed, self ).__init__()
self.module = module
self.batch_first = batch_first
def forward(self, x):

 x_size= x.size()
c_in = x.contiguous().view(x_size[0] * x_size[1], *x_size[2:])
 c_out = self.module(c_in)
 r_in = c_out.view(x_size[0], x_size[1], -1)
 if self.batch_first is False:
r_in = r_in.permute(1, 0, 2)
 return r_in
class RSNAClassifier(nn.Module):
def __init__(self, hidden_size=64):
super().__init__()
self.gru = nn.GRU(len(get_stage1_columns())+1, hidden_size, bidirectional=True, batch_first=True, num_layers=2)
 self.image_predictors = TimeDistributed(nn.Linear(hidden_size*2, 1))
self.exam_predictor = nn.Linear(hidden_size*2*2, 9)
 def forward(self, img_preds, locs):
embeds = torch.cat([img_preds, locs.view(locs.shape[0], locs.shape[1], 1)], dim=2)
 embeds, _ = self.gru(embeds)
 image_preds = self.image_predictors(embeds)
 avg_pool = torch.mean(embeds, 1)
 max_pool, _ = torch.max(embeds, 1)
 conc = torch.cat([avg_pool, max_pool], 1)
 exam_pred = self.exam_predictor(conc)
 return image_preds, exam_pred
def rsna_wloss_inference(y_true_img, y_true_exam, y_pred_img, y_pred_exam, chunk_sizes):

 y_true_img, y_true_exam, y_pred_img, y_pred_exam = torch.tensor(y_true_img, dtype=torch.float32), torch.tensor(y_true_exam, dtype=torch.float32), torch.tensor(y_pred_img, dtype=torch.float32), torch.tensor(y_pred_exam, dtype=torch.float32)
 y_true_img_chunks, y_true_exam_chunks, y_pred_img_chunks, y_pred_exam_chunks = torch.split(y_true_img, chunk_sizes, dim=0), torch.split(y_true_exam, chunk_sizes, dim=0), torch.split(y_pred_img, chunk_sizes, dim=0), torch.split(y_pred_exam, chunk_sizes, dim=0)
 label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988] ).view(1, -1)
 img_w = 0.07361963
bce_func = torch.nn.BCELoss(reduction='none')
 total_loss = torch.tensor(0, dtype=torch.float32)
 total_weights = torch.tensor(0, dtype=torch.float32)
 for i,(y_true_img_, y_true_exam_, y_pred_img_, y_pred_exam_)in enumerate(zip(y_true_img_chunks, y_true_exam_chunks, y_pred_img_chunks, y_pred_exam_chunks)) :
exam_loss = bce_func(y_pred_exam_[0, :], y_true_exam_[0, :])
 exam_loss = torch.sum(exam_loss*label_w, 1)[0]
image_loss = bce_func(y_pred_img_, y_true_img_)
 img_num = chunk_sizes[i]
qi = torch.sum(y_true_img_)/img_num
image_loss = torch.sum(img_w*qi*image_loss)
 total_loss += exam_loss+image_loss
total_weights += label_w.sum() + img_w*qi*img_num
final_loss = total_loss/total_weights
return final_loss
def rsna_wloss_train(y_true_img, y_true_exam, y_pred_img, y_pred_exam, image_masks, device):
label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988] ).view(1, -1 ).to(device)
 img_w = 0.07361963
bce_func = torch.nn.BCEWithLogitsLoss(reduction='none' ).to(device)
 total_loss = torch.tensor(0, dtype=torch.float32 ).to(device)
 total_weights = torch.tensor(0, dtype=torch.float32 ).to(device)
 for i in range(y_true_img.shape[0]):
exam_loss = bce_func(y_pred_exam[i, :], y_true_exam[i, :])
 exam_loss = torch.sum(exam_loss*label_w, 1)[0]
img_mask = image_masks[i]
image_loss = bce_func(y_pred_img[i,:], y_true_img[i,:] ).flatten()
image_loss = image_loss*img_mask
img_num = torch.sum(img_mask)
 qi = torch.sum(y_true_img[i,:])/img_num
image_loss = torch.sum(img_w*qi*image_loss)
 total_loss += exam_loss+image_loss
total_weights += label_w.sum() + img_w*qi*img_num
final_loss = total_loss/total_weights
return final_loss, total_loss, total_weights
def rsna_wloss_valid(y_true_img, y_true_exam, y_pred_img, y_pred_exam, image_masks, device):
label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988] ).view(1, -1 ).to(device)
 img_w = 0.07361963
bce_func = torch.nn.BCEWithLogitsLoss(reduction='none' ).to(device)
 total_loss = torch.tensor(0, dtype=torch.float32 ).to(device)
 total_weights = torch.tensor(0, dtype=torch.float32 ).to(device)
 for i in range(y_true_img.shape[0]):
exam_loss = bce_func(y_pred_exam[i, :], y_true_exam[i, :])
 exam_loss = torch.sum(exam_loss*label_w, 1)[0]
img_mask = image_masks[i]
image_loss = bce_func(y_pred_img[i,:], y_true_img[i,:] ).flatten()
image_loss = image_loss*img_mask
img_num = torch.sum(img_mask)
 qi = torch.sum(y_true_img[i,:])/img_num
image_loss = torch.sum(img_w*qi*image_loss)
 total_loss += exam_loss+image_loss
total_weights += label_w.sum() + img_w*qi*img_num
final_loss = total_loss/total_weights
return final_loss, total_loss, total_weights
def prepare_train_dataloader(train, cv_df, train_fold, valid_fold):
train_patients = cv_df.loc[cv_df.fold.isin(train_fold), 'StudyInstanceUID'].unique()
valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()
train_ = train.loc[train.StudyInstanceUID.isin(train_patients),:].reset_index(drop=True)
 valid_ = train.loc[train.StudyInstanceUID.isin(valid_patients),:].reset_index(drop=True)
 train_ds = RSNADataset(train_, 0.0, CFG['train_img_path'], image_subsampling=True, transforms=get_train_transforms() , output_label=True)
 valid_ds = RSNADataset(valid_, 0.0, CFG['train_img_path'], image_subsampling=False, transforms=get_valid_transforms() , output_label=True)
 train_loader = torch.utils.data.DataLoader(
 train_ds,
batch_size=CFG['train_bs'],
pin_memory=True,
drop_last=False,
shuffle=True,
num_workers=CFG['num_workers'], 
)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=1,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=True, 
)
 return train_loader, val_loader
def train_one_epoch(epoch, model, device, scaler, optimizer, train_loader):
model.train()
t = time.time()
loss_sum = 0
loss_w_sum = 0
for step,(imgs, per_image_preds, locs, image_labels, exam_label, image_masks)in enumerate(train_loader):
imgs = imgs.to(device ).float()
per_image_preds = per_image_preds.to(device ).float()
locs = locs.to(device ).float()
image_masks = image_masks.to(device ).float()
image_labels = image_labels.to(device ).float()
exam_label = exam_label.to(device ).float()
with autocast() :
image_preds, exam_pred = model(per_image_preds, locs)
 loss, total_loss, total_weights = rsna_wloss_train(image_labels, exam_label, image_preds, exam_pred, image_masks, device)
 scaler.scale(loss ).backward()
loss_sum += total_loss.detach().item()
loss_w_sum += total_weights.detach().item()
if(( step + 1)% CFG['accum_iter'] == 0)or(( step + 1)== len(train_loader)) :
scaler.step(optimizer)
 scaler.update()
optimizer.zero_grad()
if(( step + 1)% CFG['verbose_step'] == 0)or(( step + 1)== len(train_loader)) :
print(
 f'epoch {epoch} train step {step+1}/{len(train_loader)}, ' + \
f'loss: {loss_sum/loss_w_sum:.4f}, ' + \
f'time: {(time.time() - t):.4f}', end= '\r' if(step + 1)!= len(train_loader)else '
' 
)
 def post_process(exam_pred, image_pred):
rv_lv_ratio_lt_1_ix = CFG['exam_target_cols'].index('rv_lv_ratio_lt_1')
 rv_lv_ratio_gte_1_ix = CFG['exam_target_cols'].index('rv_lv_ratio_gte_1')
 central_pe_ix = CFG['exam_target_cols'].index('central_pe')
 rightsided_pe_ix = CFG['exam_target_cols'].index('rightsided_pe')
 leftsided_pe_ix = CFG['exam_target_cols'].index('leftsided_pe')
 acute_and_chronic_pe_ix = CFG['exam_target_cols'].index('acute_and_chronic_pe')
 chronic_pe_ix = CFG['exam_target_cols'].index('chronic_pe')
 negative_exam_for_pe_ix = CFG['exam_target_cols'].index('negative_exam_for_pe')
 indeterminate_ix = CFG['exam_target_cols'].index('indeterminate')
 has_pe_image = torch.max(image_pred, 1)[0][0] > 0
rv_lv_ratios = exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix]]
rv_lv_ratios_1_a = nn.functional.softmax(rv_lv_ratios, dim=1)
 rv_lv_ratios_1_a = torch.log(rv_lv_ratios_1_a/(1-rv_lv_ratios_1_a))
exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix]] = torch.where(has_pe_image, rv_lv_ratios_1_a, rv_lv_ratios)
 crl_pe = exam_pred[:, [central_pe_ix, rightsided_pe_ix, leftsided_pe_ix]]
has_no_pe = torch.max(crl_pe ,1)[0] <= 0
max_val = torch.max(crl_pe, 1)[0]
crl_pe_1_b = torch.where(crl_pe==max_val, 0.0001-crl_pe+crl_pe, crl_pe)
 exam_pred[:, [central_pe_ix, rightsided_pe_ix, leftsided_pe_ix]] = torch.where(has_pe_image*has_no_pe, crl_pe_1_b, crl_pe)
 ac_pe = exam_pred[:, [acute_and_chronic_pe_ix, chronic_pe_ix]]
both_ac_ch = torch.min(ac_pe ,1)[0] > 0
ac_pe_1_c = nn.functional.softmax(ac_pe, dim=1)
 ac_pe_1_c = torch.log(ac_pe_1_c/(1-ac_pe_1_c))
exam_pred[:, [acute_and_chronic_pe_ix, chronic_pe_ix]] = torch.where(has_pe_image*both_ac_ch, ac_pe_1_c, ac_pe)
 neg_ind = exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]]
neg_ind_1d = torch.clamp(neg_ind, max=0)
 exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]] = torch.where(has_pe_image, neg_ind_1d, neg_ind)
 ne_inde = exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]]
ne_inde_2_a = nn.functional.softmax(ne_inde, dim=1)
 ne_inde_2_a = torch.log(ne_inde_2_a/(1-ne_inde_2_a))
exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]] = torch.where(~has_pe_image, ne_inde_2_a, ne_inde)
 all_other_exam_labels = exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix,
central_pe_ix, rightsided_pe_ix, leftsided_pe_ix,
acute_and_chronic_pe_ix, chronic_pe_ix]]
all_other_exam_labels_2_b = torch.clamp(all_other_exam_labels, max=0)
 exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix,
central_pe_ix, rightsided_pe_ix, leftsided_pe_ix,
acute_and_chronic_pe_ix, chronic_pe_ix]] = torch.where(~has_pe_image, all_other_exam_labels_2_b, all_other_exam_labels)
 return exam_pred, image_pred
def valid_one_epoch(epoch, model, device, scheduler, val_loader, schd_loss_update=False):
model.eval()
t = time.time()
loss_sum = 0
loss_w_sum = 0
for step,(imgs, per_image_preds, locs, image_labels, exam_label, image_masks)in enumerate(val_loader):
imgs = imgs.to(device ).float()
per_image_preds = per_image_preds.to(device ).float()
locs = locs.to(device ).float()
image_masks = image_masks.to(device ).float()
image_labels = image_labels.to(device ).float()
exam_label = exam_label.to(device ).float()
image_preds, exam_pred = model(per_image_preds, locs)
 exam_pred, image_preds= post_process(exam_pred, image_preds)
 loss, total_loss, total_weights = rsna_wloss_valid(image_labels, exam_label, image_preds, exam_pred, image_masks, device)
 loss_sum += total_loss.detach().item()
loss_w_sum += total_weights.detach().item()
if(( step + 1)% CFG['verbose_step'] == 0)or(( step + 1)== len(val_loader)) :
print(
 f'epoch {epoch} valid Step {step+1}/{len(val_loader)}, ' + \
f'loss: {loss_sum/loss_w_sum:.4f}, ' + \
f'time: {(time.time() - t):.4f}', end='\r' if(step + 1)!= len(val_loader)else '
' 
)
 if schd_loss_update:
scheduler.step(loss_sum/loss_w_sum)
 else:
scheduler.step()
def check_label_consistency(checking_df):
df = checking_df.copy()
print(df.shape)
 df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID'] ).pe_present_on_image.max())
 df_pos = df.loc[df.positive_images_in_exam > 0.5]
df_neg = df.loc[df.positive_images_in_exam <= 0.5]
rule1a = df_pos.loc[(( df_pos.rv_lv_ratio_lt_1 > 0.5)& 
(df_pos.rv_lv_ratio_gte_1 > 0.5)) | 
(( df_pos.rv_lv_ratio_lt_1 <= 0.5)& 
(df_pos.rv_lv_ratio_gte_1 <= 0.5)) ].reset_index(drop = True)
 rule1a['broken_rule'] = '1a'
rule1b = df_pos.loc[(df_pos.central_pe <= 0.5)& 
(df_pos.rightsided_pe <= 0.5)& 
(df_pos.leftsided_pe <= 0.5)].reset_index(drop = True)
 rule1b['broken_rule'] = '1b'
rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5)& 
(df_pos.chronic_pe > 0.5)].reset_index(drop = True)
 rule1c['broken_rule'] = '1c'
rule1d = df_pos.loc[(df_pos.indeterminate > 0.5)| 
(df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)
 rule1d['broken_rule'] = '1d'
rule2a = df_neg.loc[(( df_neg.indeterminate > 0.5)& 
(df_neg.negative_exam_for_pe > 0.5)) | 
(( df_neg.indeterminate <= 0.5)& 
(df_neg.negative_exam_for_pe <= 0.5)) ].reset_index(drop = True)
 rule2a['broken_rule'] = '2a'
rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1 > 0.5)| 
(df_neg.rv_lv_ratio_gte_1 > 0.5)| 
(df_neg.central_pe > 0.5)| 
(df_neg.rightsided_pe > 0.5)| 
(df_neg.leftsided_pe > 0.5)| 
(df_neg.acute_and_chronic_pe > 0.5)| 
(df_neg.chronic_pe > 0.5)].reset_index(drop = True)
 rule2b['broken_rule'] = '2b'
errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)
 print('label in-consistency counts:', errors.shape)
 if errors.shape[0] > 0:
print(errors.broken_rule.value_counts())
 print(errors)
 assert False
def inference(model, device, df, root_path):
model.eval()
t = time.time()
ds = RSNADataset(df, 0.0, root_path, image_subsampling=False, transforms=get_valid_transforms() , output_label=False)
 dataloader = torch.utils.data.DataLoader(
 ds,
batch_size=1,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=True, 
)
 patients = ds.get_patients()
res_dfs = []
for step,(imgs, per_image_preds, locs, img_num, index, seq_ix)in enumerate(dataloader):
imgs = imgs.to(device ).float()
per_image_preds = per_image_preds.to(device ).float()
locs = locs.to(device ).float()
index = index.detach().numpy() [0]
seq_ix = seq_ix.detach().numpy() [0,:]
patient_filt =(df.StudyInstanceUID == patients[index])
 patient_df = pd.DataFrame()
patient_df['SOPInstanceUID'] = df.loc[patient_filt, 'SOPInstanceUID'].values[seq_ix]
patient_df['SeriesInstanceUID'] = df.loc[patient_filt, 'SeriesInstanceUID'].values
patient_df['StudyInstanceUID'] = patients[index]
for c in CFG['image_target_cols']+CFG['exam_target_cols']:
patient_df[c] = 0.0
image_preds, exam_pred = model(per_image_preds, locs)
 exam_pred, image_preds = post_process(exam_pred, image_preds)
 exam_pred = torch.sigmoid(exam_pred ).cpu().detach().numpy()
image_preds = torch.sigmoid(image_preds ).cpu().detach().numpy()
patient_df[CFG['exam_target_cols']] = exam_pred[0]
patient_df[CFG['image_target_cols']] = image_preds[0,:]
res_dfs += [patient_df]


if(( step + 1)% CFG['verbose_step'] == 0)or(( step + 1)== len(dataloader)) :
print(
 f'Inference Step {step+1}/{len(dataloader)}, ' + \
f'time: {(time.time() - t):.4f}', end='\r' if(step + 1)!= len(dataloader)else '
' 
)
 res_dfs = pd.concat(res_dfs, axis=0 ).reset_index(drop=True)
 res_dfs = df[['SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID']].merge(res_dfs, on=['SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID'], how='left')
 print(res_dfs[CFG['image_target_cols']+CFG['exam_target_cols']].head(5))
print(res_dfs[CFG['image_target_cols']+CFG['exam_target_cols']].tail(5))
assert res_dfs.shape[0] == df.shape[0]
check_label_consistency(res_dfs)
 return res_dfs
STAGE1_CFGS = [
{
'tag': 'efb0_stage1',
'model_constructor': RSNAImgClassifierSingle,
'dataset_constructor': RSNADatasetStage1,
'output_len': 1
},
{
'tag': 'efb0_stage1_multilabel',
'model_constructor': RSNAImgClassifier,
'dataset_constructor': RSNADatasetStage1,
'output_len': 9
},
]
STAGE1_CFGS_TAG = 'efb0-stage1-single-multi-label'
def get_stage1_columns() :
new_feats = []
for cfg in STAGE1_CFGS:
for i in range(cfg['output_len']):
f = cfg['tag']+'_'+str(i)
 new_feats += [f]
return new_feats
def update_stage1_oof_preds(df, cv_df):
res_file_name = STAGE1_CFGS_TAG+"-train.csv"
new_feats = get_stage1_columns()
for f in new_feats:
df[f] = 0
if os.path.isfile(res_file_name):
df = pd.read_csv(res_file_name)
 print('img acc:',(( df[new_feats[0]]>0)==df[CFG['image_target_cols'][0]] ).mean())
 return df
for fold,(train_fold, valid_fold)in enumerate(zip(CFG['train_folds'], CFG['valid_folds'])) :
if fold < 0:
continue
valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()
filt = df.StudyInstanceUID.isin(valid_patients)
 valid_ = df.loc[filt,:].reset_index(drop=True)
 image_preds_all_list = []
for cfg in STAGE1_CFGS:
valid_ds = cfg['dataset_constructor'](valid_, 0.0, CFG['train_img_path'], image_subsampling=False, transforms=get_valid_transforms() , output_label=True)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=256,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False,
sampler=SequentialSampler(valid_ds)
)
 device = torch.device(CFG['device'])
 model = cfg['model_constructor']().to(device)
 model.load_state_dict(torch.load('{}/model_fold_{}_{}'.format(CFG['model_path'], fold, cfg['tag'])))
 model.eval()
image_preds_all = []
correct_count = 0
count = 0
for step,(imgs, target)in enumerate(val_loader):
imgs = imgs.to(device ).float()
target = target.to(device ).float()
image_preds = model(imgs)
 if len(image_preds.shape)== 1:
image_preds = image_preds.view(-1, 1)
 correct_count +=(( image_preds[:,0]>0)== target[:,0] ).sum().detach().item()
count += imgs.shape[0]
image_preds_all += [image_preds.cpu().detach().numpy() ]
print('acc: {:.4f}, {}, {}, {}/{}'.format(correct_count/count, correct_count, count, step+1, len(val_loader)) , end='\r')
 print()
image_preds_all = np.concatenate(image_preds_all, axis=0)
 image_preds_all_list += [image_preds_all]
del model, val_loader
torch.cuda.empty_cache()
image_preds_all_list = np.concatenate(image_preds_all_list, axis=1)
 df.loc[filt, new_feats] = image_preds_all_list
df.to_csv(res_file_name, index=False)
 return df
def update_stage1_test_preds(df):
new_feats = get_stage1_columns()
for f in new_feats:
df[f] = 0
image_preds_all_list = []
for cfg in STAGE1_CFGS:
test_ds = cfg['dataset_constructor'](df, 0.0, CFG['test_img_path'], image_subsampling=False, transforms=get_valid_transforms() , output_label=False)
 test_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=256,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False,
sampler=SequentialSampler(test_ds)
)
 device = torch.device(CFG['device'])
 model = cfg['model_constructor']().to(device)
 model.load_state_dict(torch.load('{}/model_{}'.format(CFG['model_path'], cfg['tag'])))
 model.eval()
image_preds_all = []
for step, imgs in enumerate(tqdm(test_loader)) :
imgs = imgs.to(device ).float()
image_preds = model(imgs)
 image_preds_all += [image_preds.cpu().detach().numpy() ]
image_preds_all = np.concatenate(image_preds_all, axis=0)
 image_preds_all_list += [image_preds_all]
del model, test_loader
torch.cuda.empty_cache()
image_preds_all_list = np.concatenate(image_preds_all_list, axis=1)
 df.loc[:,new_feats] = image_preds_all_list
return df
if __name__ == '__main__':
if CFG['train']:
seed_everything(SEED)
 if CFG['train']:
train_df = pd.read_csv(CFG['train_path'])
 cv_df = pd.read_csv(CFG['cv_fold_path'])
 with torch.no_grad() :
train_df = update_stage1_oof_preds(train_df,cv_df)
 else:
test_df = pd.read_csv(CFG['test_path'])
 DEBUG =(test_df.shape[0]==146853)
 DEBUG
if DEBUG:
test_df = test_df.head(500)
 test_df = test_df.reset_index(drop=True)
 print(test_df.shape)
 with torch.no_grad() :
test_df = update_stage1_test_preds(test_df)
 if CFG['train']:
for fold,(train_fold, valid_fold)in enumerate(zip(CFG['train_folds'], CFG['valid_folds'])) :
train_loader, val_loader = prepare_train_dataloader(train_df, cv_df, train_fold, valid_fold)
 device = torch.device(CFG['device'])
 model = RSNAClassifier().to(device)
 scaler = GradScaler()
optimizer = torch.optim.Adam(model.parameters() , lr=CFG['lr'])
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=1); schd_loss_update=False
for epoch in range(CFG['epochs']):
train_one_epoch(epoch, model, device, scaler, optimizer, train_loader)
 with torch.no_grad() :
valid_one_epoch(epoch, model, device, scheduler, val_loader, schd_loss_update=schd_loss_update)
 torch.save(model.state_dict() ,'{}/model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag']))
model.load_state_dict(torch.load('{}/model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag'])))
 valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()
valid_ = train_df.loc[train_df.StudyInstanceUID.isin(valid_patients),:].reset_index(drop=True)
 with torch.no_grad() :
val_pred_df = inference(model, device, valid_, CFG['train_img_path'])
 target = valid_[CFG['image_target_cols']].values
pred =(val_pred_df[CFG['image_target_cols']].values > 0.5 ).astype(int)
 print('Image PE Accuracy: {:.3f}'.format(( target==pred ).mean() *100))
loss = rsna_wloss_inference(valid_[CFG['image_target_cols']].values, valid_[CFG['exam_target_cols']].values,
val_pred_df[CFG['image_target_cols']].values, val_pred_df[CFG['exam_target_cols']].values,
list(valid_.groupby('StudyInstanceUID', sort=False)['SOPInstanceUID'].count()))
print('Validation loss = {:.4f}'.format(loss.detach().item()))
del model, optimizer, train_loader, val_loader, scaler, scheduler
torch.cuda.empty_cache()
train_loader, val_loader = prepare_train_dataloader(train_df, cv_df, np.arange(0, 20), np.array([]))
device = torch.device(CFG['device'])
 model = RSNAClassifier().to(device)
 scaler = GradScaler()
optimizer = torch.optim.Adam(model.parameters() , lr=CFG['lr'])
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=1); schd_loss_update=False
for epoch in range(CFG['epochs']):
train_one_epoch(epoch, model, device, scaler, optimizer, train_loader)
 torch.save(model.state_dict() ,'{}/model_{}'.format(CFG['model_path'], CFG['tag']))
else:
device = torch.device(CFG['device'])
 model = RSNAClassifier().to(device)
 model.load_state_dict(torch.load('{}/model_{}'.format(CFG['model_path'], CFG['tag'])))
 test_pred_df = inference(model, device, test_df, CFG['test_img_path'])
 test_pred_df.to_csv('kh_submission_raw.csv')
 ids = []
labels = []
gp_mean = test_pred_df.loc[:, ['StudyInstanceUID']+CFG['exam_target_cols']].groupby('StudyInstanceUID', sort=False ).mean()
for col in CFG['exam_target_cols']:
ids += [[patient+'_'+col for patient in gp_mean.index]]
labels += [gp_mean[col].values]
ids += [test_pred_df.SOPInstanceUID.values]
labels += [test_pred_df[CFG['image_target_cols']].values[:,0]]
ids = np.concatenate(ids)
 labels = np.concatenate(labels)
 assert len(ids)== len(labels)
 submission = pd.DataFrame()
submission['id'] = ids
submission['label'] = labels
print(submission.head(3))
print(submission.tail(3))
print(submission.shape)
 submission.to_csv('submission.csv', index=False )<import_modules>
import itertools
package_path = '.. /input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'
bash_commands = [
'cp.. /input/gdcm-conda-install/gdcm.tar.',
'tar -xvzf gdcm.tar',
'conda install --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2'
]
for bashCommand in bash_commands:
process = subprocess.Popen(bashCommand.split() , stdout=subprocess.PIPE)
 output, error = process.communicate()
do_full = False
TTA_STEPS=3
CFG = {
'train': False,
'train_img_path': '.. /input/rsna-str-pulmonary-embolism-detection/train',
'test_img_path': '.. /input/rsna-str-pulmonary-embolism-detection/test',
'cv_fold_path': '.. /input/stratified-validation-strategy/rsna_train_splits_fold_20.csv',
'train_path': '.. /input/rsna-str-pulmonary-embolism-detection/train.csv',
'test_path': '.. /input/rsna-str-pulmonary-embolism-detection/test.csv',
'image_target_cols': [
'pe_present_on_image',
],
'exam_target_cols': [
'negative_exam_for_pe',
'rv_lv_ratio_gte_1',
'rv_lv_ratio_lt_1',
'leftsided_pe',
'chronic_pe',
'rightsided_pe',
'acute_and_chronic_pe',
'central_pe',
'indeterminate'
],
'img_num': 200,
'img_size': 256,
'lr': 0.0005,
'epochs': 2,
'device': 'cuda',
'train_bs': 2,
'accum_iter': 8,
'verbose_step': 1,
'num_workers': 4,
'efbnet': 'efficientnet-b0',
'train_folds': [np.arange(0,16),
np.concatenate([np.arange(0,12), np.arange(16,20)]),
np.concatenate([np.arange(0,8), np.arange(12,20)]),
np.concatenate([np.arange(0,4), np.arange(8,20)]),
np.arange(4,20),
],
'valid_folds': [np.arange(16,20),
np.arange(12,16),
np.arange(8,12),
np.arange(4,8),
np.arange(0,4)
 ],
'model_path': '.. /input/kh-rsna-model',
'tag': 'efb0_stage2_multilabel'
}
SEED = 42321
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def window(img, WL=50, WW=350):
upper, lower = WL+WW//2, WL-WW//2
X = np.clip(img.copy() , lower, upper)
 X = X - np.min(X)
 X = X / np.max(X)
 return X
def get_img(path, transforms):
d = pydicom.read_file(path)


 img =(d.pixel_array * d.RescaleSlope)+ d.RescaleIntercept
r = window(img, -600, 1500)
 g = window(img, 100, 700)
 b = window(img, 40, 400)
 res = np.concatenate([r[:, :, np.newaxis],
g[:, :, np.newaxis],
b[:, :, np.newaxis]], axis=-1)
 res = transforms(image=(res*255.0 ).astype('uint8')) ['image']
res = torch.div(res, 255.)
return res
def tta_augmentation() :
transforms = [
albu.Resize(256, 256),
albu.CLAHE(p=0.3),
albu.RandomBrightnessContrast(p=0.3),
albu.HueSaturationValue(p=0.3),
albu.ShiftScaleRotate(p=0.4),
ToTensorV2(p=1.) ,
]
return albu.Compose(transforms)
 def get_valid_transforms() :
return albu.Compose([
albu.Resize(256, 256),
ToTensorV2(p=1.) ,
], p=1.)
class RSNADatasetStage1(Dataset):
def __init__(
 self, df, label_smoothing, data_root,
image_subsampling=True, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.label_smoothing = label_smoothing
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df[CFG['image_target_cols']].values[index]
path = "{}/{}/{}/{}.dcm".format(self.data_root,
self.df.iloc[index]['StudyInstanceUID'],
self.df.iloc[index]['SeriesInstanceUID'],
self.df.iloc[index]['SOPInstanceUID'])
 img = get_img(path, self.transforms)
 return img
class RSNADataset(Dataset):
def __init__(
 self, df, label_smoothing, data_root,
image_subsampling=True, transforms=None, output_label=True 
):
super().__init__()
self.df = df
self.patients = self.df['StudyInstanceUID'].unique()
self.image_subsampling = image_subsampling
self.label_smoothing = label_smoothing
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def get_patients(self):
return self.patients
def __len__(self):
return len(self.patients)
 def __getitem__(self, index: int):
patient = self.patients[index]
df_ = self.df.loc[self.df.StudyInstanceUID == patient]
per_image_feats = get_stage1_columns()
if self.image_subsampling:
img_num = min(CFG['img_num'], df_.shape[0])
 img_ix = np.random.choice(np.arange(df_.shape[0]), replace=False, size=img_num)
 imgs = np.zeros(( CFG['img_num'],), np.float32)
 per_image_preds = np.zeros(( CFG['img_num'], len(per_image_feats)) , np.float32)
 locs = np.zeros(( CFG['img_num'],), np.float32)
 image_masks = np.zeros(( CFG['img_num'],), np.float32)
 image_masks[:img_num] = 1.
 if self.output_label:
exam_label = df_[CFG['exam_target_cols']].values[0]
image_labels = np.zeros(( CFG['img_num'], len(CFG['image_target_cols'])) , np.float32)
 else:
img_num = df_.shape[0]
img_ix = np.arange(df_.shape[0])
 imgs = np.zeros(( img_num,), np.float32)
 per_image_preds = np.zeros(( img_num, len(per_image_feats)) , np.float32)
 locs = np.zeros(( img_num,), np.float32)
 image_masks = np.zeros(( img_num,), np.float32)
 image_masks[:img_num] = 1.
 if self.output_label:
exam_label = df_[CFG['exam_target_cols']].values[0]
image_labels = np.zeros(( img_num, len(CFG['image_target_cols'])) , np.float32)
 for i, im_ix in enumerate(img_ix):
path = "{}/{}/{}/{}.dcm".format(self.data_root,
df_['StudyInstanceUID'].values[im_ix],
df_['SeriesInstanceUID'].values[im_ix],
df_['SOPInstanceUID'].values[im_ix])
 d = pydicom.read_file(path)
 locs[i] = d.ImagePositionPatient[2]
per_image_preds[i,:] = df_[per_image_feats].values[im_ix,:]
if self.output_label == True:
image_labels[i] = df_[CFG['image_target_cols']].values[im_ix]
seq_ix = np.argsort(locs)

locs = locs[seq_ix]
locs[1:img_num] = locs[1:img_num]-locs[0:img_num-1]
locs[0] = 0
per_image_preds = per_image_preds[seq_ix]
if self.output_label == True:
image_labels = image_labels[seq_ix]
image_labels = np.clip(image_labels, self.label_smoothing, 1 - self.label_smoothing)
 exam_label = np.clip(exam_label, self.label_smoothing, 1 - self.label_smoothing)
 return imgs, per_image_preds, locs, image_labels, exam_label, image_masks
else:
return imgs, per_image_preds, locs, img_num, index, seq_ix
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout 
)
 class RNSAImageFeatureExtractor(nn.Module):
def __init__(self):
super().__init__()
self.cnn_model = EfficientNet.from_name(CFG['efbnet'])
 self.pooling = nn.AdaptiveAvgPool2d(1)
 def get_dim(self):
return self.cnn_model._fc.in_features
def forward(self, x):
feats = self.cnn_model.extract_features(x)
 return self.pooling(feats ).view(x.shape[0], -1)
 class RSNAImgClassifierSingle(nn.Module):
def __init__(self):
super().__init__()
self.cnn_model = RNSAImageFeatureExtractor()
self.image_predictors = nn.Linear(self.cnn_model.get_dim() , 1)
 def forward(self, imgs):
imgs_embdes = self.cnn_model(imgs)
 image_preds = self.image_predictors(imgs_embdes)
 return image_preds
class RSNAImgClassifier(nn.Module):
def __init__(self):
super().__init__()
self.cnn_model = RNSAImageFeatureExtractor()
self.image_predictors = nn.Linear(self.cnn_model.get_dim() , 9)
 def forward(self, imgs):
imgs_embdes = self.cnn_model(imgs)
 image_preds = self.image_predictors(imgs_embdes)
 return image_preds
def post_process(exam_pred, image_pred):
rv_lv_ratio_lt_1_ix = CFG['exam_target_cols'].index('rv_lv_ratio_lt_1')
 rv_lv_ratio_gte_1_ix = CFG['exam_target_cols'].index('rv_lv_ratio_gte_1')
 central_pe_ix = CFG['exam_target_cols'].index('central_pe')
 rightsided_pe_ix = CFG['exam_target_cols'].index('rightsided_pe')
 leftsided_pe_ix = CFG['exam_target_cols'].index('leftsided_pe')
 acute_and_chronic_pe_ix = CFG['exam_target_cols'].index('acute_and_chronic_pe')
 chronic_pe_ix = CFG['exam_target_cols'].index('chronic_pe')
 negative_exam_for_pe_ix = CFG['exam_target_cols'].index('negative_exam_for_pe')
 indeterminate_ix = CFG['exam_target_cols'].index('indeterminate')
 has_pe_image = torch.max(image_pred, 1)[0][0] > 0
rv_lv_ratios = exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix]]
rv_lv_ratios_1_a = nn.functional.softmax(rv_lv_ratios, dim=1)
 rv_lv_ratios_1_a = torch.log(rv_lv_ratios_1_a/(1-rv_lv_ratios_1_a))
exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix]] = torch.where(has_pe_image, rv_lv_ratios_1_a, rv_lv_ratios)
 crl_pe = exam_pred[:, [central_pe_ix, rightsided_pe_ix, leftsided_pe_ix]]
has_no_pe = torch.max(crl_pe ,1)[0] <= 0
max_val = torch.max(crl_pe, 1)[0]
crl_pe_1_b = torch.where(crl_pe==max_val, 0.0001-crl_pe+crl_pe, crl_pe)
 exam_pred[:, [central_pe_ix, rightsided_pe_ix, leftsided_pe_ix]] = torch.where(has_pe_image*has_no_pe, crl_pe_1_b, crl_pe)
 ac_pe = exam_pred[:, [acute_and_chronic_pe_ix, chronic_pe_ix]]
both_ac_ch = torch.min(ac_pe ,1)[0] > 0
ac_pe_1_c = nn.functional.softmax(ac_pe, dim=1)
 ac_pe_1_c = torch.log(ac_pe_1_c/(1-ac_pe_1_c))
exam_pred[:, [acute_and_chronic_pe_ix, chronic_pe_ix]] = torch.where(has_pe_image*both_ac_ch, ac_pe_1_c, ac_pe)
 neg_ind = exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]]
neg_ind_1d = torch.clamp(neg_ind, max=0)
 exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]] = torch.where(has_pe_image, neg_ind_1d, neg_ind)
 ne_inde = exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]]
ne_inde_2_a = nn.functional.softmax(ne_inde, dim=1)
 ne_inde_2_a = torch.log(ne_inde_2_a/(1-ne_inde_2_a))
exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]] = torch.where(~has_pe_image, ne_inde_2_a, ne_inde)
 all_other_exam_labels = exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix,
central_pe_ix, rightsided_pe_ix, leftsided_pe_ix,
acute_and_chronic_pe_ix, chronic_pe_ix]]
all_other_exam_labels_2_b = torch.clamp(all_other_exam_labels, max=0)
 exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix,
central_pe_ix, rightsided_pe_ix, leftsided_pe_ix,
acute_and_chronic_pe_ix, chronic_pe_ix]] = torch.where(~has_pe_image, all_other_exam_labels_2_b, all_other_exam_labels)
 return exam_pred, image_pred
def check_label_consistency(checking_df):
df = checking_df.copy()
print(df.shape)
 df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID'] ).pe_present_on_image.max())
 df_pos = df.loc[df.positive_images_in_exam > 0.5]
df_neg = df.loc[df.positive_images_in_exam <= 0.5]
rule1a = df_pos.loc[(( df_pos.rv_lv_ratio_lt_1 > 0.5)& 
(df_pos.rv_lv_ratio_gte_1 > 0.5)) | 
(( df_pos.rv_lv_ratio_lt_1 <= 0.5)& 
(df_pos.rv_lv_ratio_gte_1 <= 0.5)) ].reset_index(drop = True)
 rule1a['broken_rule'] = '1a'
rule1b = df_pos.loc[(df_pos.central_pe <= 0.5)& 
(df_pos.rightsided_pe <= 0.5)& 
(df_pos.leftsided_pe <= 0.5)].reset_index(drop = True)
 rule1b['broken_rule'] = '1b'
rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5)& 
(df_pos.chronic_pe > 0.5)].reset_index(drop = True)
 rule1c['broken_rule'] = '1c'
rule1d = df_pos.loc[(df_pos.indeterminate > 0.5)| 
(df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)
 rule1d['broken_rule'] = '1d'
rule2a = df_neg.loc[(( df_neg.indeterminate > 0.5)& 
(df_neg.negative_exam_for_pe > 0.5)) | 
(( df_neg.indeterminate <= 0.5)& 
(df_neg.negative_exam_for_pe <= 0.5)) ].reset_index(drop = True)
 rule2a['broken_rule'] = '2a'
rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1 > 0.5)| 
(df_neg.rv_lv_ratio_gte_1 > 0.5)| 
(df_neg.central_pe > 0.5)| 
(df_neg.rightsided_pe > 0.5)| 
(df_neg.leftsided_pe > 0.5)| 
(df_neg.acute_and_chronic_pe > 0.5)| 
(df_neg.chronic_pe > 0.5)].reset_index(drop = True)
 rule2b['broken_rule'] = '2b'
errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)
 print('label in-consistency counts:', errors.shape)
 if errors.shape[0] > 0:
print(errors.broken_rule.value_counts())
 print(errors)
 assert False
class TimeDistributed(nn.Module):
def __init__(self, module, batch_first=True):
super(TimeDistributed, self ).__init__()
self.module = module
self.batch_first = batch_first
def forward(self, x):

 x_size= x.size()
c_in = x.contiguous().view(x_size[0] * x_size[1], *x_size[2:])
 c_out = self.module(c_in)
 r_in = c_out.view(x_size[0], x_size[1], -1)
 if self.batch_first is False:
r_in = r_in.permute(1, 0, 2)
 return r_in
def inference(model, device, df, root_path):
model.eval()
t = time.time()
ds = RSNADataset(df, 0.0, root_path, image_subsampling=False, transforms=None, output_label=False)
 dataloader = torch.utils.data.DataLoader(
 ds,
batch_size=1,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=True, 
)
 patients = ds.get_patients()
res_dfs = []
for step,(imgs, per_image_preds, locs, img_num, index, seq_ix)in enumerate(dataloader):
imgs = imgs.to(device ).float()
per_image_preds = per_image_preds.to(device ).float()
locs = locs.to(device ).float()
index = index.detach().numpy() [0]
seq_ix = seq_ix.detach().numpy() [0,:]
patient_filt =(df.StudyInstanceUID == patients[index])
 patient_df = pd.DataFrame()
patient_df['SOPInstanceUID'] = df.loc[patient_filt, 'SOPInstanceUID'].values[seq_ix]
patient_df['SeriesInstanceUID'] = df.loc[patient_filt, 'SeriesInstanceUID'].values
patient_df['StudyInstanceUID'] = patients[index]
for c in CFG['image_target_cols']+CFG['exam_target_cols']:
patient_df[c] = 0.0
image_preds, exam_pred = model(per_image_preds, locs)
 exam_pred = torch.sigmoid(exam_pred ).cpu().detach().numpy()
image_preds = torch.sigmoid(image_preds ).cpu().detach().numpy()
patient_df[CFG['exam_target_cols']] = exam_pred[0]
patient_df[CFG['image_target_cols']] = image_preds[0,:]
res_dfs += [patient_df]
if(( step + 1)% CFG['verbose_step'] == 0)or(( step + 1)== len(dataloader)) :
print(
 f'Inference Step {step+1}/{len(dataloader)}, ' + \
f'time: {(time.time() - t):.4f}', end='\r' if(step + 1)!= len(dataloader)else '
' 
)
 res_dfs = pd.concat(res_dfs, axis=0 ).reset_index(drop=True)
 res_dfs = df[['SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID']].merge(res_dfs, on=['SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID'], how='left')
 assert res_dfs.shape[0] == df.shape[0]
return res_dfs
STAGE1_CFGS = [
{
'tag': 'efb0_stage1',
'model_constructor': RSNAImgClassifierSingle,
'dataset_constructor': RSNADatasetStage1,
'output_len': 1
},
{
'tag': 'efb0_stage1_multilabel',
'model_constructor': RSNAImgClassifier,
'dataset_constructor': RSNADatasetStage1,
'output_len': 9
},
]
STAGE1_CFGS_TAG = 'efb0-stage1-single-multi-label'
def get_stage1_columns() :
new_feats = []
for cfg in STAGE1_CFGS:
for i in range(cfg['output_len']):
f = cfg['tag']+'_'+str(i)
 new_feats += [f]
return new_feats
def update_stage1_test_preds(df):
new_feats = get_stage1_columns()
df.loc[:,new_feats] = 0
df_list=[]
for n, i in enumerate(range(TTA_STEPS)) :
if n==0:
test_ds = RSNADatasetStage1(df, 0.0, CFG['test_img_path'], image_subsampling=False, transforms=get_valid_transforms() , output_label=False)
 else:
test_ds = RSNADatasetStage1(df, 0.0, CFG['test_img_path'], image_subsampling=False, transforms=tta_augmentation() , output_label=False)
 test_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=256,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False,
sampler=SequentialSampler(test_ds)
)
 image_preds_all_list = []
models = []
for cfg in STAGE1_CFGS:
device = torch.device(CFG['device'])
 model = cfg['model_constructor']().to(device)
 model.load_state_dict(torch.load('{}/model_{}'.format(CFG['model_path'], cfg['tag'])))
 model.eval()
models.append(model)
 image_preds_all = []
for step, imgs in enumerate(tqdm(test_loader)) :
imgs = imgs.to(device ).float()
for model in models:
image_preds = model(imgs)
 image_preds_all += [image_preds.cpu().detach().numpy() ]
del test_loader
torch.cuda.empty_cache()
image_preds_all_image = np.concatenate(image_preds_all[::2], axis=0)
 image_preds_all_exam = np.concatenate(image_preds_all[1::2], axis=0)
 image_preds_all = np.concatenate([image_preds_all_image, image_preds_all_exam], axis=1)
 print(np.array(new_feats ).shape)
 print(np.array(image_preds_all ).shape)
 temp = df.copy()
temp.loc[:, new_feats] = image_preds_all
df_list.append(temp)
 del temp
return df_list
class RSNAClassifier(nn.Module):
def __init__(self, hidden_size=64):
super().__init__()
self.gru = nn.GRU(len(get_stage1_columns())+1, hidden_size, bidirectional=True, batch_first=True, num_layers=2)
 self.image_predictors = TimeDistributed(nn.Linear(hidden_size*2, 1))
self.exam_predictor = nn.Linear(hidden_size*2*2, 9)
 def forward(self, img_preds, locs):
embeds = torch.cat([img_preds, locs.view(locs.shape[0], locs.shape[1], 1)], dim=2)
 embeds, _ = self.gru(embeds)
 image_preds = self.image_predictors(embeds)
 avg_pool = torch.mean(embeds, 1)
 max_pool, _ = torch.max(embeds, 1)
 conc = torch.cat([avg_pool, max_pool], 1)
 exam_pred = self.exam_predictor(conc)
 return image_preds, exam_pred
if __name__ == '__main__':
seed_everything(SEED)
 if path.exists('.. /input/rsna-str-pulmonary-embolism-detection/train')and not do_full:
test_df = pd.read_csv(CFG['test_path'] ).head(1000)
 else:
test_df = pd.read_csv(CFG['test_path'])
 with torch.no_grad() :
test_df = update_stage1_test_preds(test_df)
 device = torch.device(CFG['device'])
 model = RSNAClassifier().to(device)
 model.load_state_dict(torch.load('{}/model_{}'.format(CFG['model_path'], CFG['tag'])))
 test_pred_df = inference(model, device, test_df[0], CFG['test_img_path'])
 for i in test_df[1:]:
test_pred_df.loc[:, test_pred_df.columns[3:]] += inference(model, device, i, CFG['test_img_path'] ).loc[:, test_pred_df.columns[3:]]
test_pred_df.loc[:, test_pred_df.columns[3:]]/=TTA_STEPS
ids = []
labels = []
gp_mean = test_pred_df.loc[:, ['StudyInstanceUID']+CFG['exam_target_cols']].groupby('StudyInstanceUID', sort=False ).mean()
for col in CFG['exam_target_cols']:
ids += [[patient+'_'+col for patient in gp_mean.index]]
labels += [gp_mean[col].values]
ids += [test_pred_df.SOPInstanceUID.values]
labels += [test_pred_df[CFG['image_target_cols']].values[:,0]]
ids = np.concatenate(ids)
 labels = np.concatenate(labels)
 assert len(ids)== len(labels)
 submission = pd.DataFrame()
submission['id'] = ids
submission['label'] = labels
print(submission.head())
 submission.to_csv('submission.csv', index=False )<load_from_zip>
param_space = {
'max_depth': [3, 5, 7],
'min_child_weight': [1.0, 2.0, 4.0]
}
!cp.. /input/gdcm-conda-install/gdcm.tar.
 !tar -xvzf gdcm.tar
!conda install --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2<import_modules>
param_combinations = itertools.product(param_space['max_depth'], param_space['min_child_weight'] )
import gdcm<set_options>
params = []
scores = []
print(os.cpu_count())
 n_gpu = torch.cuda.device_count()<set_options>
for max_depth, min_child_weight in param_combinations:
score_folds = []
kf = KFold(n_splits=4, shuffle=True, random_state=123456)
 for tr_idx, va_idx in kf.split(train_x):
tr_x, va_x = train_x.iloc[tr_idx], train_x.iloc[va_idx]
tr_y, va_y = train_y.iloc[tr_idx], train_y.iloc[va_idx]
model = XGBClassifier(n_estimators=20, random_state=71, max_depth=max_depth, min_child_weight=min_child_weight)
 model.fit(tr_x, tr_y)
 va_pred = model.predict_proba(va_x)[:, 1]
logloss = log_loss(va_y, va_pred)
 score_folds.append(logloss)
 score_mean = np.mean(score_folds)
 params.append(( max_depth, min_child_weight))
scores.append(score_mean )
def set_seeds(SEED):
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 if n_gpu > 0:
torch.cuda.manual_seed_all(SEED)
 torch.backends.cudnn.deterministic = True
set_seeds(SEED=2020 )<define_variables>
best_idx = np.argsort(scores)[0]
best_param = params[best_idx]
print(f'max_depth: {best_param[0]}, min_child_weight: {best_param[1]}' )
batch_size = 8
num_workers = os.cpu_count()<load_from_csv>
from sklearn.preprocessing import OneHotEncoder
test_df = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/test.csv')
<feature_engineering>
train_x2 = train.drop(['Survived'], axis=1)
 test_x2 = test.copy()
%%time
def get_file_paths(df, base_dir):
PATH = ".. /input/rsna-str-pulmonary-embolism-detection/"
df["file_path"] = df.apply(lambda x:(PATH + base_dir +
x["StudyInstanceUID"] + "/" +
x["SeriesInstanceUID"]+ "/" +
x["SOPInstanceUID"]+ ".dcm"), axis=1)
 return df
test_df = get_file_paths(test_df, "test/" )<load_pretrained>
train_x2 = train_x2.drop(['PassengerId'], axis=1)
 test_x2 = test_x2.drop(['PassengerId'], axis=1 )
def extract_meta_feats(file_path):
image = pydicom.dcmread(file_path)
 study_uid = image.StudyInstanceUID
series_uid = image.SeriesInstanceUID
sop_uid = image.SOPInstanceUID
image_position_patient = image.ImagePositionPatient
return [study_uid, series_uid, sop_uid, image_position_patient]<save_to_csv>
train_x2 = train_x2.drop(['Name', 'Ticket', 'Cabin'], axis=1)
 test_x2 = test_x2.drop(['Name', 'Ticket', 'Cabin'], axis=1 )
%%time
if do_all:
test_files = test_df["file_path"].values.tolist()
results = Parallel(n_jobs=-1, verbose=1 )(map(delayed(extract_meta_feats), test_files))
test_meta_df = pd.DataFrame(results, columns=[
"StudyInstanceUID", "SeriesInstanceUID", "SOPInstanceUID", "ImagePositionPatient"])
 test_meta_df.to_csv('test_metadata.csv', index=False )<set_options>
cat_cols = ['Sex', 'Embarked', 'Pclass']
ohe = OneHotEncoder(categories='auto', sparse=False)
 ohe.fit(train_x2[cat_cols].fillna('NA'))
del test_meta_df
gc.collect()<prepare_x_and_y>
ohe_columns = []
for i, c in enumerate(cat_cols):
ohe_columns += [f'{c}_{v}' for v in ohe.categories_[i]]
def load_dicom_array(file_path):
dicom = pydicom.dcmread(file_path)
 M = float(dicom.RescaleSlope)
 B = float(dicom.RescaleIntercept)
 dicom = pixel_array(dicom)
 dicom = dicom * M
dicom = dicom + B
return dicom<categorify>
ohe_train_x2 = pd.DataFrame(ohe.transform(train_x2[cat_cols].fillna('NA')) , columns=ohe_columns)
 ohe_test_x2 = pd.DataFrame(ohe.transform(test_x2[cat_cols].fillna('NA')) , columns=ohe_columns )
def get_3ch_image(image, transform=None):
image_lung = window(image, WL=-600, WW=1500 ).astype(np.float32)
 image_mediastinal = window(image, WL=40, WW=400 ).astype(np.float32)
 image_pe_specific = window(image, WL=100, WW=700 ).astype(np.float32)
 if transform:
image_lung = transform(image=image_lung)['image']
image_mediastinal = transform(image=image_mediastinal)['image']
image_pe_specific = transform(image=image_pe_specific)['image']
image_array = np.stack([image_lung, image_mediastinal, image_pe_specific], axis=-1)
 image_array = image_array.transpose(2, 0, 1)
 return image_array<categorify>
train_x2 = train_x2.drop(cat_cols, axis=1)
 test_x2 = test_x2.drop(cat_cols, axis=1 )
class ImageDataset(data.Dataset):
def __init__(self, df, img_list, img_size=512, transform=None):
self.df = df
self.img_list = img_list
self.img_size = img_size
self.transform = transform
def __len__(self):
return len(self.img_list)
 def __getitem__(self, idx: int):
row = self.df.iloc[idx]
img_path = self.img_list[idx]
img_array = get_3ch_image(load_dicom_array(img_path), self.transform)
 item = {"img": img_array,
"StudyInstanceUID": row["StudyInstanceUID"],
"SeriesInstanceUID": row["SeriesInstanceUID"],
"SOPInstanceUID": row["SOPInstanceUID"]}
return item<create_dataframe>
train_x2 = pd.concat([train_x2, ohe_train_x2], axis=1)
 test_x2 = pd.concat([test_x2, ohe_test_x2], axis=1 )
id_columns = ["StudyInstanceUID", "SeriesInstanceUID", "SOPInstanceUID"]
test_loader = data.DataLoader(ImageDataset(test_df[id_columns], img_list=test_df["file_path"].values,),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True,
drop_last=False )<choose_model_class>
num_cols = ['Age', 'SibSp', 'Parch', 'Fare']
for col in num_cols:
train_x2[col].fillna(train_x2[col].mean() , inplace=True)
 test_x2[col].fillna(train_x2[col].mean() , inplace=True )
class ResNet(nn.Module):
def __init__(self, pretrained=False,
num_classes=2):
super().__init__()
base_model = models.__getattribute__("resnet18" )(
 pretrained=pretrained)
 layers = list(base_model.children())[:-2]
layers.append(nn.AdaptiveMaxPool2d(1))
self.encoder = nn.Sequential(*layers)
 in_features = base_model.fc.in_features
self.classifier = nn.Sequential(
 nn.Linear(in_features, 512), nn.ReLU() , nn.Dropout(p=0.5),
nn.Linear(512, 2))
def forward(self, x):
batch_size = x.size(0)
 features = self.encoder(x ).view(batch_size, -1)
 x = self.classifier(features)
 return F.softmax(x, dim=1), features<set_options>
train_x2['Fare'] = np.log1p(train_x2['Fare'])
 test_x2['Fare'] = np.log1p(test_x2['Fare'] )
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 model = ResNet()<load_pretrained>
from sklearn.linear_model import LogisticRegression
checkpoint = torch.load(".. /input/0927-resnet18-loss03220/train.4.pth", map_location=device)
 model.load_state_dict(checkpoint["model_state_dict"])
 model.to(device)
 model.eval()
print("model load finish" )<train_model>
model_xgb = XGBClassifier(n_estimators=20, random_state=71)
 model_xgb.fit(train_x, train_y)
 pred_xgb = model_xgb.predict_proba(test_x)[:, 1]
print(model )<prepare_output>
model_lr = LogisticRegression(solver='lbfgs', max_iter=300)
 model_lr.fit(train_x2, train_y)
 pred_lr = model_lr.predict_proba(test_x2)[:, 1]
if do_all:
preds_df = pd.DataFrame({
"id": test_df["SOPInstanceUID"].values[:len(predictions)],
"pred" : predictions
})
 print(preds_df.head() )<set_options>
pred = pred_xgb * 0.8 + pred_lr * 0.2
pred_label = np.where(pred > 0.5, 1, 0 )
<define_variables><EOS> 
submission = pd.DataFrame({'passengerId': test['PassengerId'], 'Survived': pred_label})
 submission.to_csv('submission_ensemble.csv', index=False )