Kaggle competition: RSNA STR Pulmonary Embolism Detection
<SOS> metric: weightedmeancolumnwiselogloss Kaggle data source: rsna-str-pulmonary-embolism-detection<categorify>
package_path = '.. /input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'
!cp.. /input/gdcm-conda-install/gdcm.tar.
 !tar -xvzf gdcm.tar
!conda install --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2
page=pd.get_dummies(new_data['homepage_end'])
 page.head()<drop_column>
CFG = {
'train': False,
'train_img_path': '.. /input/rsna-str-pulmonary-embolism-detection/train',
'test_img_path': '.. /input/rsna-str-pulmonary-embolism-detection/test',
'cv_fold_path': '.. /input/stratified-validation-strategy/rsna_train_splits_fold_20.csv',
'train_path': '.. /input/rsna-str-pulmonary-embolism-detection/train.csv',
'test_path': '.. /input/rsna-str-pulmonary-embolism-detection/test.csv',
'image_target_cols': [
'pe_present_on_image',
],
'exam_target_cols': [
'negative_exam_for_pe',
'rv_lv_ratio_gte_1',
'rv_lv_ratio_lt_1',
'leftsided_pe',
'chronic_pe',
'rightsided_pe',
'acute_and_chronic_pe',
'central_pe',
'indeterminate'
],
'img_num': 200,
'img_size': 256,
'lr': 0.0005,
'epochs': 2,
'device': 'cuda',
'train_bs': 2,
'accum_iter': 8,
'verbose_step': 1,
'num_workers': 4,
'efbnet': 'efficientnet-b6',
'train_folds': [np.arange(0,16),
np.concatenate([np.arange(0,12), np.arange(16,20)]),
np.concatenate([np.arange(0,8), np.arange(12,20)]),
np.concatenate([np.arange(0,4), np.arange(8,20)]),
np.arange(4,20),
],
'valid_folds': [np.arange(16,20),
np.arange(12,16),
np.arange(8,12),
np.arange(4,8),
np.arange(0,4)
 ],
'model_path': '.. /input/rsnab6/',
'tag': 'efb6_stage2_multilabel_v2'
}
SEED = 42321
new_data=new_data.drop('homepage',axis=1 )<drop_column>
STAGE1_CFGS = [
{
'tag': 'efb6_stage1_example',
'model_constructor': RSNAImgClassifierSingle,
'dataset_constructor': RSNADatasetStage1,
'output_len': 1
},
{
'tag': 'efb6_stage1_multilabel',
'model_constructor': RSNAImgClassifier,
'dataset_constructor': RSNADatasetStage1,
'output_len': 9
},
]
STAGE1_CFGS_TAG = 'efb6-stage1-single-multi-label'
new_data=new_data.drop('poster_path',axis=1 )<feature_engineering>
def get_stage1_columns() :
new_feats = []
for cfg in STAGE1_CFGS:
for i in range(cfg['output_len']):
f = cfg['tag']+'_'+str(i)
 new_feats += [f]
return new_feats
def update_stage1_oof_preds(df, cv_df):
res_file_name = STAGE1_CFGS_TAG+"-train.csv"
new_feats = get_stage1_columns()
for f in new_feats:
df[f] = 0
if os.path.isfile(res_file_name):
df = pd.read_csv(res_file_name)
 print('img acc:',(( df[new_feats[0]]>0)==df[CFG['image_target_cols'][0]] ).mean())
 return df
for fold,(train_fold, valid_fold)in enumerate(zip(CFG['train_folds'], CFG['valid_folds'])) :
if fold < 0:
continue
valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()
filt = df.StudyInstanceUID.isin(valid_patients)
 valid_ = df.loc[filt,:].reset_index(drop=True)
 image_preds_all_list = []
for cfg in STAGE1_CFGS:
valid_ds = cfg['dataset_constructor'](valid_, 0.0, CFG['train_img_path'], image_subsampling=False, transforms=get_valid_transforms() , output_label=True)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=256,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False,
sampler=SequentialSampler(valid_ds)
)
 device = torch.device(CFG['device'])
 model = cfg['model_constructor']().to(device)
 model.load_state_dict(torch.load('{}/model_fold_{}_{}'.format(CFG['model_path'], fold, cfg['tag'])))
 model.eval()
image_preds_all = []
correct_count = 0
count = 0
for step,(imgs, target)in enumerate(val_loader):
imgs = imgs.to(device ).float()
target = target.to(device ).float()
image_preds = model(imgs)
 if len(image_preds.shape)== 1:
image_preds = image_preds.view(-1, 1)
 correct_count +=(( image_preds[:,0]>0)== target[:,0] ).sum().detach().item()
count += imgs.shape[0]
image_preds_all += [image_preds.cpu().detach().numpy() ]
print('acc: {:.4f}, {}, {}, {}/{}'.format(correct_count/count, correct_count, count, step+1, len(val_loader)) , end='\r')
 print()
image_preds_all = np.concatenate(image_preds_all, axis=0)
 image_preds_all_list += [image_preds_all]
del model1, model2, model3, model4, model5, val_loader
torch.cuda.empty_cache()
image_preds_all_list = np.concatenate(image_preds_all_list, axis=1)
 df.loc[filt, new_feats] = image_preds_all_list
df.to_csv(res_file_name, index=False)
 return df
new_data['len_overview']=new_data['overview'].fillna('NAN' ).apply(lambda x:len(x))<sort_values>
def update_stage1_test_preds(df):
new_feats = get_stage1_columns()
for f in new_feats:
df[f] = 0
image_preds_all_list = []
MODELS = {}
for cfg in STAGE1_CFGS:
test_ds = cfg['dataset_constructor'](df, 0.0, CFG['test_img_path'], image_subsampling=False, transforms=get_valid_transforms() , output_label=False)
 test_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=64 * 2,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False,
sampler=SequentialSampler(test_ds)
)
 device = torch.device(CFG['device'])
 model0 = cfg['model_constructor']().to(device)
 model0.load_state_dict(torch.load('{}/model_fold_0_{}'.format(CFG['model_path'], cfg['tag'])))
 model0.eval()
model1 = cfg['model_constructor']().to(device)
 model1.load_state_dict(torch.load('{}/model_fold_1_{}'.format(CFG['model_path'], cfg['tag'])))
 model1.eval()
model2 = cfg['model_constructor']().to(device)
 model2.load_state_dict(torch.load('{}/model_fold_2_{}'.format(CFG['model_path'], cfg['tag'])))
 model2.eval()
model3 = cfg['model_constructor']().to(device)
 model3.load_state_dict(torch.load('{}/model_fold_3_{}'.format(CFG['model_path'], cfg['tag'])))
 model3.eval()
model4 = cfg['model_constructor']().to(device)
 model4.load_state_dict(torch.load('{}/model_fold_4_{}'.format(CFG['model_path'], cfg['tag'])))
 model4.eval()
MODELS[cfg['tag']] = [model0, model1, model2, model3, model4]
image_preds_all = []
image_preds_all_m = []
for step, imgs in enumerate(tqdm(test_loader)) :
imgs = imgs.to(device ).float()
image_preds0 = MODELS['efb6_stage1_example'][0](imgs)
 image_preds1 = MODELS['efb6_stage1_example'][1](imgs)
 image_preds2 = MODELS['efb6_stage1_example'][2](imgs)
 image_preds3 = MODELS['efb6_stage1_example'][3](imgs)
 image_preds4 = MODELS['efb6_stage1_example'][4](imgs)
 image_preds = torch.stack([image_preds0,
image_preds1,
image_preds2,
image_preds3,
image_preds4,
] ).mean(dim=0)
 image_preds_all += [image_preds.cpu().detach().numpy() ]
image_preds0_m = MODELS['efb6_stage1_multilabel'][0](imgs)
 image_preds1_m = MODELS['efb6_stage1_multilabel'][1](imgs)
 image_preds2_m = MODELS['efb6_stage1_multilabel'][2](imgs)
 image_preds3_m = MODELS['efb6_stage1_multilabel'][3](imgs)
 image_preds4_m = MODELS['efb6_stage1_multilabel'][4](imgs)
 image_preds_m = torch.stack([image_preds0_m,
image_preds1_m,
image_preds2_m,
image_preds3_m,
image_preds4_m,
] ).mean(dim=0)
 image_preds_all_m += [image_preds_m.cpu().detach().numpy() ]
image_preds_all = np.concatenate(image_preds_all, axis=0)
 image_preds_all_m = np.concatenate(image_preds_all_m, axis=0)
 image_preds_all_list = [image_preds_all, image_preds_all_m]
del model0, model1, model2, model3, model4, test_loader, MODELS
torch.cuda.empty_cache()
image_preds_all_list = np.concatenate(image_preds_all_list, axis=1)
 df.loc[:,new_feats] = image_preds_all_list
return df
len_rew_sort=new_data['len_overview'].sort_values(ascending=True)
 len_rew_sort.head()<define_variables>
seed_everything(SEED)
 test_df = pd.read_csv(CFG['test_path'])
 DEBUG =(test_df.shape[0]==146853)
 if DEBUG:
if getpass.getuser() == 'robmulla':
CFG['num_workers'] = 32
pass
else:
test_studies = test_df['StudyInstanceUID'].unique().tolist()
test_df = test_df.loc[test_df['StudyInstanceUID'].isin(test_studies[:25])]
else:
public_sub = pd.read_csv('.. /input/rsna-submission-files/submission_b6_no_checks.csv')
 ss = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/sample_submission.csv')
 private_ss = ss.loc[~ss['id'].isin(public_sub['id'])]
private_studies = private_ss.loc[private_ss['id'].str.len() != 12]['id'].str[:12].unique()
private_sop = private_ss.loc[private_ss['id'].str.len() == 12]['id'].unique()
test_df = test_df.loc[
test_df['StudyInstanceUID'].isin(private_studies)&
test_df['SOPInstanceUID'].isin(private_sop)
 ].reset_index(drop=True )
length=len(new_data['len_overview'])
 m=0.1
n=0.1
arr_len_ove=[]
for i in range(1,11):
arr_len_ove.append(round(length*m))
m+=n
arr_len_ove<filter>
with torch.no_grad() :
test_df = update_stage1_test_preds(test_df )
for i in range(10):
qu=qu_arr[i]
if i==0:
new_data['len_overview'].loc[(new_data['len_overview']<len_rew_sort.iloc[arr_len_ove[i]-1])]=qu
else:
new_data['len_overview'].loc[(new_data['len_overview']<len_rew_sort.iloc[arr_len_ove[i]-1])&(new_data['len_overview'] >qu_arr[i-1])]=qu
print(i,qu )<groupby>
device = torch.device(CFG['device'])
 model = RSNAClassifier().to(device)
 test_pred_dfs = []
model.load_state_dict(torch.load(f"{CFG['model_path']}/model_{CFG['tag']}"))
test_pred_df = inference(model, device, test_df, CFG['test_img_path'])
 test_pred_dfs.append(test_pred_df )
len_ove_agg=new_data.groupby('len_overview' ).revenue.aggregate(['min','max','std'] )<feature_engineering>
test_pred_df.to_csv('submission_raw.csv' )
new_data['geres_name']=new_data['genres'].str.findall(r''name'\s?:\s?'(\w+)'')
 new_data['geres_name'].head()<drop_column>
ids = []
labels = []
gp_mean = test_pred_df.loc[:, ['StudyInstanceUID']+CFG['exam_target_cols']].groupby('StudyInstanceUID', sort=False ).mean()
for col in CFG['exam_target_cols']:
ids += [[patient+'_'+col for patient in gp_mean.index]]
labels += [gp_mean[col].values]
ids += [test_pred_df.SOPInstanceUID.values]
labels += [test_pred_df[CFG['image_target_cols']].values[:,0]]
ids = np.concatenate(ids)
 labels = np.concatenate(labels )
new_data=new_data.drop('genres',axis=1 )<find_best_params>
assert len(ids)== len(labels)
 submission = pd.DataFrame()
submission['id'] = ids
submission['label'] = labels
country=new_data['production_countries'].str.findall(r'[A-Z]{2,5}')
<feature_engineering>
if not DEBUG:
submission = pd.concat([public_sub,
submission],
).reset_index(drop=True )
<feature_engineering><EOS> 
print(submission.head(3))
print(submission.tail(3))
print(submission.shape)
 submission.to_csv('submission.csv', index=False )