Kaggle competition: RSNA STR Pulmonary Embolism Detection
for col in ['title','overview', 'tagline','original_title']:
train_df['len_' + col] = train_df[col].fillna('' ).apply(lambda x: len(str(x)))
 train_df['words_' + col] = train_df[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
train_df = train_df.drop(col, axis=1)
 test_df['len_' + col] = test_df[col].fillna('' ).apply(lambda x: len(str(x)))
 test_df['words_' + col] = test_df[col].fillna('' ).apply(lambda x: len(str(x.split(' '))))
test_df = test_df.drop(col, axis=1 )<categorify>
USE_POST_PROCESS = True
train_df = pd.concat([train_df, pd.get_dummies(train_df['release_date_weekday'], prefix='release_weekday')], axis=1)
 train_df = pd.concat([train_df, pd.get_dummies(train_df['release_date_month'], prefix='release_month')], axis=1)
 train_df = pd.concat([train_df, pd.get_dummies(train_df['release_date_quarter'], prefix='release_quarter')], axis=1)
 test_df = pd.concat([test_df, pd.get_dummies(test_df['release_date_weekday'], prefix='release_weekday')], axis=1)
 test_df = pd.concat([test_df, pd.get_dummies(test_df['release_date_month'], prefix='release_month')], axis=1)
 test_df = pd.concat([test_df, pd.get_dummies(test_df['release_date_quarter'], prefix='release_quarter')], axis=1 )<feature_engineering>
PATH_CSV = '.. /input/rsna-str-pulmonary-embolism-detection/test.csv'
PATH_CSV_SUB = '.. /input/rsna-str-pulmonary-embolism-detection/sample_submission.csv'
PATH_FOLDER = '.. /input/rsna-str-pulmonary-embolism-detection/test'
USE_FOLDS = [0, 2]
PATH_STAGE1_ARGS_LIST = ['.. /input/rsna-emboli-ckpts/stage1/fold{}_ResNext101.json'.format(i)for i in USE_FOLDS]
PATH_STAGE1_MODEL_LIST = ['.. /input/rsna-emboli-ckpts/stage1/fold{}_ResNext101.pt'.format(i)for i in USE_FOLDS]
PATH_STAGE2_ARGS_LIST = ['.. /input/rsna-emboli-ckpts/stage2_pseudo/F{}GRUPseudo.json'.format(i)for i in USE_FOLDS]
PATH_STAGE2_MODEL_LIST = ['.. /input/rsna-emboli-ckpts/stage2_pseudo/F{}GRUPseudo.pt'.format(i)for i in USE_FOLDS]
TX_STAGE2_ARGS = '.. /input/rsna-emboli-ckpts/args_tx.json'
TX_STAGE2_MODEL = '.. /input/rsna-emboli-ckpts/fold_0_epoch_12_txGRUall.pt'
PATH_KH = '.. /input/submission-kh3/kh/weight_files'
def new_features(df):
df['budget_to_popularity'] = df['budget'] / df['popularity']
df['budget_to_runtime'] = df['budget'] / df['runtime']
df['_budget_year_ratio'] = df['budget'] /(df['release_date_year'] * df['release_date_year'])
 df['_releaseYear_popularity_ratio'] = df['release_date_year'] / df['popularity']
df['_releaseYear_popularity_ratio2'] = df['popularity'] / df['release_date_year']
df['runtime_to_mean_year'] = df['runtime'] / df.groupby("release_date_year")["runtime"].transform('mean')
 df['popularity_to_mean_year'] = df['popularity'] / df.groupby("release_date_year")["popularity"].transform('mean')
 df['budget_to_mean_year'] = df['budget'] / df.groupby("release_date_year")["budget"].transform('mean')
 return df<feature_engineering>
class CTData(torch.utils.data.Dataset):
def __init__(self, path_folder, uids):
self.path_folder = path_folder
self.uids = uids
def get_pixels_hu(self, slices):
image = np.stack([s.pixel_array for s in slices])
 image = image.astype(np.int16)
 for slice_number in range(len(slices)) :
intercept = slices[slice_number].RescaleIntercept
slope = slices[slice_number].RescaleSlope
if slope != 1:
image[slice_number] = slope \
* image[slice_number].astype(np.float64)
 image[slice_number] = image[slice_number].astype(np.int16)
 image[slice_number] += np.int16(intercept)
 image = np.array(image, dtype=np.int16)
 image[image < -1024] = -1024
return image
def get_pixels_kaggle(self, slices:list)->(np.array, list):
image = np.stack([(s.pixel_array * s.RescaleSlope)+ s.RescaleIntercept for s in slices])
 return image.astype('float32')
 def load_scan(self, path_to_dir, order_by_instance_number=False):
dicom_paths = glob.glob(path_to_dir + '/*/*.dcm')
 dicom_slices = [dicom.read_file(x, force=True)for x in dicom_paths]
zipped = list(zip(dicom_slices, dicom_paths))
if order_by_instance_number:
zipped.sort(key = lambda x:int(x[0].InstanceNumber))
else:
zipped.sort(key = lambda x: float(x[0].ImagePositionPatient[2]))
dicom_slices = list(list(zip(*zipped)) [0])
 dicom_paths = list(list(zip(*zipped)) [1])
 return dicom_slices, dicom_paths
def windowing(self, img, mode="pe"):
if mode == "mediastinal":
WL = 40
WW = 450
elif mode == "pe":
WL = 100
WW = 700
elif mode == "lung":
WL = -600
WW = 1500
upper, lower = WL + WW // 2, WL - WW // 2
X = np.clip(img.copy() , lower, upper)
 X = X - lower
X = X /(upper - lower)
 return X
def mk_preprocess(self, image):

 img = image[:, 32:480, 32:480]
img0 = self.windowing(img, "mediastinal")[np.newaxis]
img1 = self.windowing(img, "pe")[np.newaxis]
img2 = self.windowing(img, "lung")[np.newaxis]
img = np.concatenate(( img0, img1, img2), axis=0)
 img = img.astype(np.float32)
 return img
def __len__(self):
return len(self.uids)
 def __getitem__(self, index):

 t0 = time.time()
uid = self.uids[index]
path_to_dcm = os.path.join(self.path_folder, self.uids[index])
 dicom_slices, dicom_paths = self.load_scan(path_to_dcm)
 dicom_pixels = self.get_pixels_hu(dicom_slices)
 dicom_pixels_mk = self.mk_preprocess(dicom_pixels)
 dicom_pixels_kh = self.get_pixels_kaggle(dicom_slices)
 img_num = len(dicom_slices)
 locs = np.zeros(( img_num,), np.float32)
 for i in range(img_num):
locs[i] = dicom_slices[i].ImagePositionPatient[2]
seq_ix = np.argsort(locs)
 locs = locs[seq_ix]
locs[1:img_num] = locs[1:img_num]-locs[0:img_num-1]
locs[0] = 0
locs = locs.astype('float32')
 dicom_pixels = dicom_pixels.astype('float32')
 dicom_pixels_kh = kh_model_all.preprocess(dicom_pixels_kh)
 print('one item time: ', time.time() - t0)
 return uid, dicom_paths, dicom_pixels_mk, dicom_pixels_kh, locs
train_df =new_features(train_df)
 test_df =new_features(test_df)
 train_df.head()<prepare_x_and_y>
class FeatSeqALLModel(object):
def __init__(self, stage1_args_path, stage1_model_path, stage2_args_path, stage2_model_path, tx_args_path=None, tx_model_path=None):
super(FeatSeqALLModel, self ).__init__()
self.has_tx = False
if tx_args_path is not None:
self.has_tx = True
with open(stage1_args_path, "rb")as j:
stage1_args = json.load(j)
 stage1_args = Namespace(**stage1_args)
 self.stage1_model = slicenet.ResNext(stage1_args, use_pretrained=False)
 stage1_load_checkpoint = torch.load(stage1_model_path)
 self.stage1_model.load_state_dict(stage1_load_checkpoint["model"], strict=True)
 self.stage1_model.embedder.fc = torch.nn.Identity()
self.stage1_model.cuda()
with open(stage2_args_path, "rb")as j:
stage2_args = json.load(j)
 stage2_args = Namespace(**stage2_args)
 self.stage2_model = featseqallnet.FeatGRUALL(stage2_args, use_pretrained=False)
 stage2_load_checkpoint = torch.load(stage2_model_path)
 self.stage2_model.load_state_dict(stage2_load_checkpoint["model"], strict=True)
 self.stage2_model.cuda()
if self.has_tx:
with open(tx_args_path, "rb")as j:
tx_args = json.load(j)
 tx_args = Namespace(**tx_args)
 self.tx_model = featseqallnet.FeatTxGruALL(tx_args, use_pretrained=False)
 tx_load_checkpoint = torch.load(tx_model_path)
 self.tx_model.load_state_dict(tx_load_checkpoint["model"], strict=True)
 self.tx_model.cuda()
def predict(self, pixel_arrays, batch_size=64, verbosity=0):
self.stage1_model.eval()
with torch.no_grad() :
image_feats = []
f_len = pixel_arrays.shape[1]
batch_images = []
for center_loc in range(f_len):
h5_file = np.array(pixel_arrays[:, center_loc, :, :])
 image = h5_file
batch_images.append(image)
 if(len(batch_images)== batch_size)or(center_loc ==(f_len-1)) :
batch = torch.from_numpy(np.array(batch_images))
batch = batch.cuda()
result = self.stage1_model(batch)
 image_feats.extend(result.detach().cpu().numpy())
 if verbosity:
pass
torch.cuda.empty_cache()
batch_images = []
self.stage2_model.eval()
with torch.no_grad() :
feats = torch.from_numpy(np.array(image_feats)[np.newaxis])
 feats = feats.cuda()
results = self.stage2_model(feats)
 logits = results[0,:,0]
exam_logits = logits[:9]
image_logits = logits[9:]
if self.has_tx:
self.tx_model.eval()
with torch.no_grad() :
tx_results = self.tx_model(feats)
 tx_logits = tx_results[0,:,0]
tx_exam_logits = tx_logits[:9]
tx_image_logits = tx_logits[9:]
exam_logits =(exam_logits + tx_exam_logits)/ 2
image_logits =(image_logits + tx_image_logits)/ 2
torch.cuda.empty_cache()
return exam_logits, image_logits
y = train_df['log_revenue']
X = train_df.drop(['id', 'genres', 'homepage', 'imdb_id','original_language', 'poster_path', 'production_companies', 'production_countries', 'spoken_languages',
'Keywords', 'cast', 'crew', 'revenue', 'collection_name','all_genres', 'all_production_companies', 'all_production_countries', 'all_spoken_languages',
'all_Keywords', 'all_cast', 'all_crew', 'release_date', 'revenue', 'log_revenue', 'budget', 'release_date_quarter',
'release_date_day', 'release_date_weekofyear', 'release_date_month', 'release_date_weekday',
'budget_to_runtime'
], axis = 1)
 X_test= test_df.drop(['id', 'genres', 'homepage', 'imdb_id','original_language', 'poster_path', 'production_companies', 'production_countries', 'spoken_languages',
'Keywords', 'cast', 'crew', 'collection_name','all_genres', 'all_production_companies', 'all_production_countries', 'all_spoken_languages',
'all_Keywords', 'all_cast', 'all_crew', 'release_date', 'budget', 'release_date_quarter', 'release_date_day',
'release_date_weekofyear', 'release_date_month', 'release_date_weekday',
'budget_to_runtime'
], axis = 1)
 X.head()<split>
start_time = time.time()
feat_seq_all_model_list = [FeatSeqALLModel(
 stage1_args_path=PATH_STAGE1_ARGS_LIST[0],
stage1_model_path=PATH_STAGE1_MODEL_LIST[0],
stage2_args_path=PATH_STAGE2_ARGS_LIST[0],
stage2_model_path=PATH_STAGE2_MODEL_LIST[0],
tx_args_path=TX_STAGE2_ARGS,
tx_model_path=TX_STAGE2_MODEL),
FeatSeqALLModel(
 stage1_args_path=PATH_STAGE1_ARGS_LIST[1],
stage1_model_path=PATH_STAGE1_MODEL_LIST[1],
stage2_args_path=PATH_STAGE2_ARGS_LIST[1],
stage2_model_path=PATH_STAGE2_MODEL_LIST[1])]
print(time.time() - start_time, ' s for loading models' )
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=1 )<choose_model_class>
def sigmoid(x):
x = np.array(x)
 return 1 /(1 +np.exp(-x))
kh_model_all = kh_model.KHModel(
 weight_dir = PATH_KH,
fold = None
)
simple_forest = RandomForestRegressor(criterion = 'mse',
n_estimators =100,
random_state = 1 )<predict_on_test>
def post_process(exam_logits, image_logits, epsilon=1e-15):
if exam_logits[0] > 0.5:
exam_logits[1:] = np.clip(exam_logits[1:], epsilon, 0.5-epsilon)
 image_logits = np.clip(image_logits, epsilon, 0.5-epsilon)
 elif exam_logits[1] > 0.5:
image_logits = np.clip(image_logits, epsilon, 0.5-epsilon)
 else:
if image_logits.max() <= 0.5:
image_logits[np.argmax(image_logits)] = 0.5+epsilon
if exam_logits[7] > 0.5 and exam_logits[8] > 0.5:
if exam_logits[7] >= exam_logits[8]:
exam_logits[8] = np.clip(exam_logits[8], epsilon, 0.5-epsilon)
 else:
exam_logits[7] = np.clip(exam_logits[7], epsilon, 0.5-epsilon)
 if exam_logits[7] <= 0.5 and exam_logits[8] <= 0.5:
if exam_logits[7] >= exam_logits[8]:
exam_logits[7] = 0.5+epsilon
else:
exam_logits[8] = 0.5+epsilon
if exam_logits[4] <= 0.5 and exam_logits[5] <= 0.5 and exam_logits[6] <= 0.5:
idx_max = np.argmax([exam_logits[4], exam_logits[5], exam_logits[6]])
 if idx_max == 0:
exam_logits[4] = 0.5+epsilon
elif idx_max == 1:
exam_logits[5] = 0.5+epsilon
else:
exam_logits[6] = 0.5+epsilon
if exam_logits[2] > 0.5 and exam_logits[3] > 0.5:
if exam_logits[2] >= exam_logits[3]:
exam_logits[3] = np.clip(exam_logits[3], 0, 0.5-epsilon)
 else:
exam_logits[2] = np.clip(exam_logits[2], 0, 0.5-epsilon)
 return exam_logits, image_logits
simple_forest.fit(X_train, y_train)
 simple_forest_pred = simple_forest.predict(X_valid )<compute_test_metric>
df_test = pd.read_csv(PATH_CSV)
 print('StudyInstanceUID :', len(df_test.StudyInstanceUID.drop_duplicates()))
test_uids = list(df_test.StudyInstanceUID.drop_duplicates())
 DEBUG =(df_test.shape[0]==146853)
 if DEBUG:
test_uids = test_uids[:3]
print('Is DEBUG: ', DEBUG )
mse = mean_squared_error(simple_forest_pred, y_valid)
 mse<save_to_csv>
target_cols_kh = ['negative_exam_for_pe', 'rv_lv_ratio_gte_1', 'rv_lv_ratio_lt_1',
'leftsided_pe', 'chronic_pe', 'rightsided_pe', 'acute_and_chronic_pe',
'central_pe', 'indeterminate']
exam_target_cols = ['negative_exam_for_pe', 'indeterminate', 'chronic_pe', 'acute_and_chronic_pe',
'central_pe', 'leftsided_pe', 'rightsided_pe', 'rv_lv_ratio_gte_1', 'rv_lv_ratio_lt_1']
reorder_idxs = [0, 8, 4, 6, 7, 3, 5, 1, 2]
[target_cols_kh[i] for i in reorder_idxs] == exam_target_cols
simple_forest_pred = simple_forest.predict(X_test)
 simple_forest_rev = np.expm1(simple_forest_pred)
 os.chdir('.. /working')
 submit = pd.DataFrame({'Id': test_df['id'], 'revenue': simple_forest_rev})
 submit.to_csv('submission_simple_forest.csv', index=False)
 submit.head()<choose_model_class>
df_save = {}
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
light_gbm = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1, random_state=1)
 light_gbm.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200 )<load_pretrained>
ctdata = CTData(path_folder = PATH_FOLDER, uids = test_uids)
 ctloader = torch.utils.data.DataLoader(ctdata, shuffle=False, batch_size=1, num_workers=1, pin_memory=False)
 t0 = time.time()
for uid, dicom_paths, pixel_arrays_mk, pixel_arrays_kh, locs in tqdm(ctloader):
uid = uid[0]
dps = [dp[0] for dp in dicom_paths]
uid_iids = [dp.split('/')[-1][:-4] for dp in dps]
pixel_arrays_mk = np.array(pixel_arrays_mk)[0]
t00 = time.time()
for mi, feat_seq_all_model in enumerate(feat_seq_all_model_list):
uid_exam_logits, uid_image_logits = feat_seq_all_model.predict(pixel_arrays_mk, batch_size=64, verbosity=DEBUG)
 mi_uid_exam_preds = torch.sigmoid(uid_exam_logits ).detach().cpu().numpy()
mi_uid_image_preds = torch.sigmoid(uid_image_logits ).detach().cpu().numpy()
if mi == 0:
uid_exam_preds_list = mi_uid_exam_preds[:, np.newaxis]
uid_image_preds_list = mi_uid_image_preds[:, np.newaxis]
else:
uid_exam_preds_list = np.concatenate(( uid_exam_preds_list, mi_uid_exam_preds[:, np.newaxis]), axis=1)
 uid_image_preds_list = np.concatenate(( uid_image_preds_list, mi_uid_image_preds[:, np.newaxis]), axis=1)
 print('one inference time: ', time.time() - t00)
 uid_exam_preds_mk = uid_exam_preds_list[:, 0] * 2/3 + uid_exam_preds_list[:, 1] * 1/3
uid_image_preds_mk = uid_image_preds_list[:, 0] * 2/3 + uid_image_preds_list[:, 1] * 1/3
pixel_arrays_kh = pixel_arrays_kh[0]
locs = np.expand_dims(np.array(locs), -1)
 result_image6, result_exam6 = kh_model_all.predict(pixel_arrays_kh, locs, batch_size=32)
 result_image = np.mean([result_image6], axis=0)
 result_exam = np.mean([result_exam6], axis=0)
 result_image_bh = sigmoid(result_image)
 result_exam_bh = sigmoid(result_exam)
 uid_exam_preds_bh = result_exam_bh[reorder_idxs]
uid_image_preds_bh = result_image_bh[:, 0]
uid_exam_preds = uid_exam_preds_mk * 2/3 + uid_exam_preds_bh * 1/3
uid_image_preds = uid_image_preds_mk * 2/3 + uid_image_preds_bh* 1/3
print(uid_exam_preds)
 print(uid_image_preds[:20])
 if USE_POST_PROCESS:
uid_exam_preds, uid_image_preds = post_process(uid_exam_preds, uid_image_preds)
 for(exam_label, exam_label_pred)in zip(exam_target_cols, uid_exam_preds):
df_save[uid+'_'+exam_label] = exam_label_pred
for(iid, iid_pred)in zip(uid_iids, uid_image_preds):
df_save[iid] = iid_pred
print('Total Time: ', time.time() - t0, 'seconds' )
eli5.show_weights(light_gbm, feature_filter=lambda x: x != '<BIAS>' )<save_to_csv>
df_save = pd.DataFrame(df_save.items() , columns=['id', 'label'])
 print(df_save.shape )
light_gbm_pred = light_gbm.predict(X_test)
 light_gbm_rev = np.expm1(light_gbm_pred)
 os.chdir('.. /working')
 submit = pd.DataFrame({'Id': test_df['id'], 'revenue': light_gbm_rev})
 submit.to_csv('submission_light_gbm.csv', index=False)
 submit.head()<define_search_space>
df_sub = pd.read_csv(PATH_CSV_SUB)
 print(df_sub.shape)
 df_sub = pd.merge(df_sub[['id']], df_save, on='id', how='left')
 df_sub.fillna(0.5, inplace=True)
 df_sub.to_csv("submission.csv", index=None )
<choose_model_class><EOS> 
df_sub.iloc[5847:5880]