Kaggle competition: RSNA STR Pulmonary Embolism Detection
<SOS> metric: weightedmeancolumnwiselogloss Kaggle data source: rsna-str-pulmonary-embolism-detection<feature_engineering>
!cp.. /input/gdcm-conda-install/gdcm.tar.
 !tar -xvzf gdcm.tar
!conda install --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2
df_input['log_budget'] = np.log10(df_input['budget'] )<feature_engineering>
import gdcm
df_input['budget/popularity1'] = df_input['budget']/df_input['popularity']
df_input['budget/popularity2'] = df_input['budget']/df_input['popularity2']
df_input['budget/runtime'] = df_input['budget']/df_input['runtime']<feature_engineering>
print(os.cpu_count())
 n_gpu = torch.cuda.device_count()
df_input['_popularity_mean_year']=df['popularity']/df.groupby("release_year")["popularity"].transform('mean')
 df_input['_budget_runtime_ratio']=df['budget']/df['runtime']
df_input['_budget_popularity_ratio']=df['budget']/df['popularity']
df_input['_budget_year_ratio']=df['budget']/(df['release_year']*df['release_year'])
 df_input['_releaseYear_popularity_ratio']=df['release_year']/df['popularity']
df_input['_releaseYear_popularity_ratio2']=df['popularity']/df['release_year']
df_input['_popularity_totalVotes_ratio']=df['totalVotes']/df['popularity']
df_input['_rating_popularity_ratio']=df['rating']/df['popularity']
df_input['_rating_totalVotes_ratio']=df['totalVotes']/df['rating']
df_input['_totalVotes_releaseYear_ratio']=df['totalVotes']/df['release_year']
df_input['_budget_rating_ratio']=df['budget']/df['rating']
df_input['_runtime_rating_ratio']=df['runtime']/df['rating']
df_input['_budget_totalVotes_ratio']=df['budget']/df['totalVotes']<feature_engineering>
def set_seeds(SEED):
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 if n_gpu > 0:
torch.cuda.manual_seed_all(SEED)
 torch.backends.cudnn.deterministic = True
set_seeds(SEED=2020 )
cols = df_input.loc[:, df_input.isnull().sum() >0].columns
df_input.loc[:, cols] = df_input[cols].fillna(df_input[cols].mean() )<load_pretrained>
batch_size = 8
num_workers = os.cpu_count()
with open('df_input.pkl', 'wb')as f:
pickle.dump(df_input , f )<data_type_conversions>
test_df = pd.read_csv('.. /input/rsna-str-pulmonary-embolism-detection/test.csv')
no_numeric = df_input.apply(lambda s:pd.to_numeric(s, errors='coerce')).isnull().all()
no_numeric[no_numeric]<rename_columns>
%%time
def get_file_paths(df, base_dir):
PATH = ".. /input/rsna-str-pulmonary-embolism-detection/"
df["file_path"] = df.apply(lambda x:(PATH + base_dir +
x["StudyInstanceUID"] + "/" +
x["SeriesInstanceUID"]+ "/" +
x["SOPInstanceUID"]+ ".dcm"), axis=1)
 return df
test_df = get_file_paths(test_df, "test/" )
df_input.columns = ["".join(c if c.isalnum() else "_" for c in str(x)) for x in df_input.columns]<count_missing_values>
def extract_meta_feats(file_path):
image = pydicom.dcmread(file_path)
 study_uid = image.StudyInstanceUID
series_uid = image.SeriesInstanceUID
sop_uid = image.SOPInstanceUID
image_position_patient = image.ImagePositionPatient
return [study_uid, series_uid, sop_uid, image_position_patient]
[df_input.isnull().sum() ]<prepare_x_and_y>
%%time
if do_all:
test_files = test_df["file_path"].values.tolist()
results = Parallel(n_jobs=-1, verbose=1 )(map(delayed(extract_meta_feats), test_files))
test_meta_df = pd.DataFrame(results, columns=[
"StudyInstanceUID", "SeriesInstanceUID", "SOPInstanceUID", "ImagePositionPatient"])
 test_meta_df.to_csv('test_metadata.csv', index=False )
X_all = df_input
X_all.drop([0],inplace = True)
 y_all = df["ln_revenue"]
y_all.index = X_all.index<drop_column>
del test_meta_df
gc.collect()
X_all.drop(columns = ['budget'],inplace = True )<drop_column>
def load_dicom_array(file_path):
dicom = pydicom.dcmread(file_path)
 M = float(dicom.RescaleSlope)
 B = float(dicom.RescaleIntercept)
 dicom = pixel_array(dicom)
 dicom = dicom * M
dicom = dicom + B
return dicom
<split>
def get_3ch_image(image, transform=None):
image_lung = window(image, WL=-600, WW=1500 ).astype(np.float32)
 image_mediastinal = window(image, WL=40, WW=400 ).astype(np.float32)
 image_pe_specific = window(image, WL=100, WW=700 ).astype(np.float32)
 if transform:
image_lung = transform(image=image_lung)['image']
image_mediastinal = transform(image=image_mediastinal)['image']
image_pe_specific = transform(image=image_pe_specific)['image']
image_array = np.stack([image_lung, image_mediastinal, image_pe_specific], axis=-1)
 image_array = image_array.transpose(2, 0, 1)
 return image_array
test_X = X_all.iloc[3000:]<import_modules>
class ImageDataset(data.Dataset):
def __init__(self, df, img_list, img_size=512, transform=None):
self.df = df
self.img_list = img_list
self.img_size = img_size
self.transform = transform
def __len__(self):
return len(self.img_list)
 def __getitem__(self, idx: int):
row = self.df.iloc[idx]
img_path = self.img_list[idx]
img_array = get_3ch_image(load_dicom_array(img_path), self.transform)
 item = {"img": img_array,
"StudyInstanceUID": row["StudyInstanceUID"],
"SeriesInstanceUID": row["SeriesInstanceUID"],
"SOPInstanceUID": row["SOPInstanceUID"]}
return item
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import StandardScaler<split>
id_columns = ["StudyInstanceUID", "SeriesInstanceUID", "SOPInstanceUID"]
test_loader = data.DataLoader(ImageDataset(test_df[id_columns], img_list=test_df["file_path"].values,),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True,
drop_last=False )
train_X, val_X, train_y, val_y = train_test_split(X_all.iloc[:3000],
y_all.iloc[:3000],
test_size=0.25, random_state=1 )<choose_model_class>
class ResNet(nn.Module):
def __init__(self, pretrained=False,
num_classes=2):
super().__init__()
base_model = models.__getattribute__("resnet18" )(
 pretrained=pretrained)
 layers = list(base_model.children())[:-2]
layers.append(nn.AdaptiveMaxPool2d(1))
self.encoder = nn.Sequential(*layers)
 in_features = base_model.fc.in_features
self.classifier = nn.Sequential(
 nn.Linear(in_features, 512), nn.ReLU() , nn.Dropout(p=0.5),
nn.Linear(512, 2))
def forward(self, x):
batch_size = x.size(0)
 features = self.encoder(x ).view(batch_size, -1)
 x = self.classifier(features)
 return F.softmax(x, dim=1), features
random_seed = 2019
k = 10
fold = list(KFold(k, shuffle = True, random_state = random_seed ).split(train))
np.random.seed(random_seed )<init_hyperparams>
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 model = ResNet()
xgb = XGBRegressor() 
<choose_model_class>
checkpoint = torch.load(".. /input/0927-resnet18-loss03220/train.4.pth", map_location=device)
 model.load_state_dict(checkpoint["model_state_dict"])
 model.to(device)
 model.eval()
print("model load finish" )
xgb_model = XGBRegressor(max_depth=4,
min_child_weight=4,
learning_rate=0.03,
n_estimators=500,
objective='reg:linear',
nthread = 4,
gamma=1.3,
silent=1,
subsample=0.7,
colsample_bytree=0.3,
colsample_bylevel=0.5)
 xgb_model.fit(train_X,train_y)
 xgb_prediction = xgb_model.predict(val_X)
 xgb_rmse = mean_squared_error(val_y, xgb_prediction )<compute_test_metric>
print(model )
math.sqrt(xgb_rmse )<predict_on_test>
if do_all:
preds_df = pd.DataFrame({
"id": test_df["SOPInstanceUID"].values[:len(predictions)],
"pred" : predictions
})
 print(preds_df.head() )
xgb_pred = xgb_model.predict(test_X )<create_dataframe>
del test_loader, model
gc.collect()
pred_xgb = pd.DataFrame(np.exp(xgb_pred)-1,columns=["revenue"])
 pred_xgb<define_variables>
label_cols = [
"negative_exam_for_pe",
"rv_lv_ratio_gte_1",
"rv_lv_ratio_lt_1",
"leftsided_pe",
"chronic_pe",
"rightsided_pe",
"acute_and_chronic_pe",
"central_pe",
"indeterminate",
]
n_classes = 9
batch_size = 8
num_workers = os.cpu_count()
lr = 1e-5
lrgamma = 0.95
DECAY = 0.0
test_id = test["id"]<save_to_csv>
tstmdf = pd.read_csv('test_metadata.csv' )
sub=pd.concat([test_id, pred_xgb],axis=1)
 sub.to_csv('TMDB_xgb.csv',index=False )<choose_model_class>
%%time
tstmdf['SliceID'] = tstmdf[['SeriesInstanceUID', 'StudyInstanceUID']].apply(
 lambda x: '{}__{}'.format(*x.tolist()), 1 )
lgbm = LGBMRegressor() 
<choose_model_class>
%%time
poscols = ['ImagePos{}'.format(i)for i in range(1, 4)]
tstmdf[poscols] = pd.DataFrame(tstmdf['ImagePositionPatient']\ 
.apply(lambda x: list(map(float, ast.literal_eval(x)))).tolist())
 tstmdf = tstmdf.sort_values(['SliceID']+poscols)\
[['StudyInstanceUID', 'SliceID', 'SOPInstanceUID']+poscols].reset_index(drop=True )
lgbm_model = LGBMRegressor(n_estimators= 500,
objection ='regressor',
metric='rmse',
max_depth = 2,
num_leaves=10,
min_child_samples=500,
learning_rate=0.01,
boosting = 'gbdt',
num_iterations = 1500,
min_data_in_leaf= 10,
bagging_freq = 1,
bagging_fraction = 0.9,
feature_fraction = 0.7,
importance_type='gain',
use_best_model=True)
 lgbm_model.fit(train_X, train_y)
 lgbm_prediction = lgbm_model.predict(val_X)
 lgbm_rmse = mean_squared_error(val_y, lgbm_prediction )<compute_test_metric>
tstmdf['seq'] =(tstmdf.groupby(['SliceID'] ).cumcount() + 1 )
math.sqrt(lgbm_rmse )<predict_on_test>
keepcols = ['StudyInstanceUID', 'SliceID', 'SOPInstanceUID', 'seq']
tstmdf = tstmdf[keepcols]
lgbm_pred = lgbm_model.predict(test_X )<create_dataframe>
%%time
test_emb_f = np.load("emb_test_embdim512.npz")
 test_emb = test_emb_f["embeddings"]
tstdf = pd.DataFrame({
"StudyInstanceUID": test_emb_f["StudyInstanceUID"],
"SeriesInstanceUID": test_emb_f["SeriesInstanceUID"],
"SOPInstanceUID": test_emb_f["SOPInstanceUID"]} )
pred_lgbm = pd.DataFrame(np.exp(lgbm_pred)-1,columns=["revenue"])
 pred_lgbm<save_to_csv>
tstdf = tstdf[["SOPInstanceUID"]].merge(tstmdf, on="SOPInstanceUID", how="left" )
sub=pd.concat([test_id, pred_lgbm],axis=1)
 sub.to_csv('TMDB_lgbm.csv',index=False )<choose_model_class>
tstdf['embidx'] = range(tstdf.shape[0] )
<choose_model_class>
del test_df, tstmdf
gc.collect()
cat_model = CatBoostRegressor(iterations=2000,
learning_rate=0.01,
depth=6,
eval_metric='RMSE',
colsample_bylevel=0.6,
bagging_temperature = 0.1,
metric_period = None,
early_stopping_rounds=200)
 cat_model.fit(train_X, train_y)
 cat_prediction = cat_model.predict(val_X)
 cat_rmse = mean_squared_error(val_y, cat_prediction )<compute_test_metric>
def collatefn(batch):
maxlen = max([l['emb'].shape[0] for l in batch])
 embdim = batch[0]['emb'].shape[1]
withlabel = 'labels' in batch[0]
for b in batch:
masklen = maxlen-len(b['emb'])
 b['emb'] = np.vstack(( np.zeros(( masklen, embdim)) , b['emb']))
b['embidx'] = torch.cat(( torch.ones(( masklen),dtype=torch.long)*-1, b['embidx']))
b['mask'] = np.ones(( maxlen))
b['mask'][:masklen] = 0.
 outbatch = {'emb' : torch.tensor(np.vstack([np.expand_dims(b['emb'], 0)\
for b in batch])).float() }
outbatch['mask'] = torch.tensor(np.vstack([np.expand_dims(b['mask'], 0)\
for b in batch])).float()
outbatch['embidx'] = torch.tensor(np.vstack([np.expand_dims(b['embidx'], 0)\
for b in batch])).float()
if withlabel:
outbatch["labels"] = torch.tensor(np.vstack([b["labels"] for b in batch])).float()
outbatch["StudyInstanceUID"] = [b["StudyInstanceUID"] for b in batch]
return outbatch
math.sqrt(cat_rmse )<predict_on_test>
test_dataset = PEDataset(tstdf, test_emb, labels=False)
 test_loader = data.DataLoader(test_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
collate_fn=collatefn )
cat_pred = cat_model.predict(test_X )<create_dataframe>
class PositionalEncoding(nn.Module):
def __init__(self, d_model, dropout=0.1, max_len=5000):
super(PositionalEncoding, self ).__init__()
self.dropout = nn.Dropout(p=dropout)
 pe = torch.zeros(max_len, d_model)
 position = torch.arange(0, max_len, dtype=torch.float ).unsqueeze(1)
 div_term = torch.exp(torch.arange(0, d_model, 2 ).float() *(-math.log(10000.0)/ d_model))
pe[:, 0::2] = torch.sin(position * div_term)
 pe[:, 1::2] = torch.cos(position * div_term)
 pe = pe.unsqueeze(0 ).transpose(0, 1)
 self.register_buffer('pe', pe)
 def forward(self, x):
x = x + self.pe[:x.size(0), :]
return self.dropout(x )
pred_cat = pd.DataFrame(np.exp(cat_pred)-1,columns=["revenue"])
 pred_cat<save_to_csv>
class PETransformerModel(nn.Module):
def __init__(self, n_classes, ninp, nhead, nhid, nlayers, dropout=0.5):
super(PETransformerModel, self ).__init__()
self.pos_encoder = PositionalEncoding(ninp, dropout)
 encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
 self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
 self.ninp = ninp
self.decoder = nn.Linear(ninp, n_classes)
 self.init_weigths()
def init_weigths(self):
initrange = 0.1
self.decoder.bias.data.zero_()
self.decoder.weight.data.uniform_(-initrange, initrange)
 def forward(self, x):
x = x * math.sqrt(self.ninp)
 x = self.pos_encoder(x)
 hidden = self.transformer_encoder(x)
 output = self.decoder(hidden.mean(1))
return output
sub=pd.concat([test_id, pred_cat],axis=1)
 sub.to_csv('TMDB_cat.csv',index=False )<define_variables>
nhid = 768
nlayers = 2
nhead = 2
dropout = 0.2
ansamble = 0.4 * pred_lgbm["revenue"] + 0.2 * pred_xgb["revenue"] + 0.4 * pred_cat["revenue"]<concatenate>
embed_size = test_emb.shape[-1] * 3
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 model = PETransformerModel(n_classes, embed_size, nhead, nhid, nlayers, dropout=dropout)
 checkpoint = torch.load(".. /input/pe-transformer-model/transformer_epoch15.pth", map_location=device)
 model.load_state_dict(checkpoint)
 model.to(device )
sub3=pd.concat([test_id, ansamble],axis=1)
 sub3<save_to_csv>
classes = [
"{}_negative_exam_for_pe",
"{}_rv_lv_ratio_gte_1",
"{}_rv_lv_ratio_lt_1",
"{}_leftsided_pe",
"{}_chronic_pe",
"{}_rightsided_pe",
"{}_acute_and_chronic_pe",
"{}_central_pe",
"{}_indeterminate",
]
sub3.to_csv('TMDB_ansamble.csv',index=False )<define_variables>
sub = pd.read_csv(".. /input/rsna-str-pulmonary-embolism-detection/sample_submission.csv")
 print(sub.shape )
<concatenate><EOS> 
%%time
preds_all_df = []
if do_all:
tqdm_loader = tqdm(test_loader)
 for step, batch in enumerate(tqdm_loader):
mask = batch['mask'].to(device, dtype=torch.int)
 inputs = batch["emb"]
inputs = inputs.to(device, dtype=torch.float, non_blocking=True)
 StudyInstanceUIDs = batch["StudyInstanceUID"]
with torch.no_grad() :
logits = model(inputs)
 preds = torch.sigmoid(logits ).detach().cpu().numpy()
for i, pred in enumerate(preds):
StudyInstanceUID = StudyInstanceUIDs[i]
id_names = [c.format(StudyInstanceUID)for c in classes]
tmp_df = pd.DataFrame({"id": id_names, "pred": pred.tolist() })
 preds_all_df.append(tmp_df)
 preds_all_df.append(preds_df)
 pred_sub = pd.concat(preds_all_df, 0)
 sub = sub.merge(pred_sub, on="id", how="left")
 sub = sub[["id", "pred"]]
sub.columns = ["id", "label"]
print(sub.isna().sum())
 sub = sub.fillna(0.5)
 sub.to_csv("submission.csv", index=False)
 else:
sub.to_csv("submission.csv", index=False )