Kaggle competition: RSNA STR Pulmonary Embolism Detection
<SOS> metric: weightedmeancolumnwiselogloss Kaggle data source: rsna-str-pulmonary-embolism-detection<data_type_conversions>
!cp /kaggle/input/gdcm-conda-install/gdcm.tar.
 !tar -xzf gdcm.tar
!conda install -q --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2
df_country =(df_country.T/df_country.sum(axis=1)).T.fillna(0 )<feature_engineering>
from fastai.vision.all import *
from fastai.medical.imaging import *
dfdic_feature["production_countries"] = select_top_n(df_country, topn=10,nmin=2)
 dfdic_feature["production_countries"].head()<import_modules>
datapath = Path("/kaggle/input/rsna-str-pulmonary-embolism-detection/")
 testdatapath = datapath/'test'
cnn2dmodelpath = Path("/kaggle/input/rsnape2dmodels/")
 cnn3dmodelpath = Path("/kaggle/input/rsnape3dmodels/")
 seqmodelpath = Path("/kaggle/input/rsnapeseqmodels/")
 test_df = pd.read_csv(datapath/'test.csv')
 sub_df = pd.read_csv(datapath/'sample_submission.csv' )
import datetime<filter>
[o for o in sub_df['id'].values if "df06fad17bc3" in o]
df[df["release_date"].isnull() ]<feature_engineering>
lung_window =(1500, -600)
 pe_window =(700, 100)
 mediastinal_window =(400, 40)
 windows =(lung_window, pe_window, mediastinal_window)
 def read_dcm_img(dcm, windows=windows):
"Read single slice in RGB"
return torch.stack([dcm.windowed(*w)for w in windows] )
df.loc[3829, "release_date"] = "5/1/00"<feature_engineering>
def get_dls(tensors, size=256, bs=128):
"Get study dataloader"
tfms = [[RandomResizedCropGPU(size, min_scale=0.9)], []]
dsets = Datasets(tensors, tfms=tfms, splits=([0,1], [2,3]))
batch_tfms = []
dls = dsets.dataloaders(bs=bs, after_batch=batch_tfms, num_workers=2)
 return dls
dls = get_dls(torch.zeros(4, 3, 512, 512), size=480, bs=32)
 dls.c = 2
df["release_year"] = pd.to_datetime(df["release_date"] ).dt.year.astype(int)
 df.loc[df["release_year"]>2020, "release_year"] = df.loc[df["release_year"]>2020, "release_year"]-100
df["release_month"] = pd.to_datetime(df["release_date"] ).dt.month.astype(int)
 df["release_day"] = pd.to_datetime(df["release_date"] ).dt.day.astype(int )<feature_engineering>
learn2d_fold0 = cnn_learner(dls, xresnet34, pretrained=False, loss_func=nn.CrossEntropyLoss() , model_dir='.')
 learn2d_fold0.path = cnn2dmodelpath
learn2d_fold0.load('xresnet34-512-PR-fold0');
df["release_date"] = df.apply(lambda s: datetime.datetime(
 year=s["release_year"],month=s["release_month"],day=s["release_day"]), axis=1 )<feature_engineering>
learn2d_fold1 = cnn_learner(dls, xresnet34, pretrained=False, loss_func=nn.CrossEntropyLoss() , model_dir='.')
 learn2d_fold1.path = cnn2dmodelpath
learn2d_fold1.load('xresnet34-512-PR-fold1');
df["release_dayofyear"] = df["release_date"].dt.dayofyear
df["release_dayofweek"] = df["release_date"].dt.dayofweek<feature_engineering>
learn2d_fold2 = cnn_learner(dls, xresnet34, pretrained=False, loss_func=nn.CrossEntropyLoss() , model_dir='.')
 learn2d_fold2.path = cnn2dmodelpath
learn2d_fold2.load('xresnet34-512-PR-fold2');
df['has_homepage'] = 1
df.loc[ pd.isnull(df['homepage']),'has_homepage'] = 0<feature_engineering>
learn2d_fold3 = cnn_learner(dls, xresnet34, pretrained=False, loss_func=nn.CrossEntropyLoss() , model_dir='.')
 learn2d_fold3.path = cnn2dmodelpath
learn2d_fold3.load('xresnet34-512-PR-fold3');
df['num_Keywords'] = df['Keywords'].apply(lambda x: len(x)if x != {} else 0 )<feature_engineering>
learn2d_fold4 = cnn_learner(dls, xresnet34, pretrained=False, loss_func=nn.CrossEntropyLoss() , model_dir='.')
 learn2d_fold4.path = cnn2dmodelpath
learn2d_fold4.load('xresnet34-512-PR-fold4');
df['overview_word_count'] = df['overview'].apply(lambda x: len(str(x ).split()))
df['overview_char_count'] = df['overview'].apply(lambda x: len(str(x)) )<feature_engineering>
model0 = learn2d_fold0.model.eval().to(device)
 model1 = learn2d_fold1.model.eval().to(device)
 model2 = learn2d_fold2.model.eval().to(device)
 model3 = learn2d_fold3.model.eval().to(device)
 model4 = learn2d_fold4.model.eval().to(device )
df['tagline_word_count'] = df['tagline'].apply(lambda x: len(str(x ).split()))
df['tagline_char_count'] = df['tagline'].apply(lambda x: len(str(x)))
 df['isTaglineNA'] = 0
df.loc[df['tagline'] == 0 ,"isTaglineNA"] = 1<feature_engineering>
from fastai.text.all import *
df['title_word_count'] = df['title'].apply(lambda x: len(str(x ).split()))
df['title_char_count'] = df['title'].apply(lambda x: len(str(x)) )<feature_engineering>
input_pad_idx = None
df['num_cast'] = df['cast'].apply(len )<feature_engineering>
class AWD_LSTM(Module):
"AWD-LSTM inspired by https://arxiv.org/abs/1708.02182"
initrange=0.1
def __init__(self, emb_sz,n_hid, n_layers, hidden_p=0.2, input_p=0.6, weight_p=0.5, bidir=False):
store_attr('emb_sz,n_hid,n_layers')
 self.bs = 1
self.n_dir = 2 if bidir else 1
self.rnns = nn.ModuleList([self._one_rnn(emb_sz if l == 0 else n_hid,(n_hid)//self.n_dir, bidir, weight_p, l)for l in range(n_layers)])
 self.input_dp = RNNDropout(input_p)
 self.hidden_dps = nn.ModuleList([RNNDropout(hidden_p)for l in range(n_layers)])
 self.reset()
def forward(self, x, from_embeds=False):
if from_embeds: inp = x
else: inp = combined_embeddings[x].to(device)
 bs,sl = inp.shape[:2]
if bs!=self.bs: self._change_hidden(bs)
 output = self.input_dp(inp)
 new_hidden = []
for l,(rnn,hid_dp)in enumerate(zip(self.rnns, self.hidden_dps)) :
output, new_h = rnn(output, self.hidden[l])
 new_hidden.append(new_h)
 if l != self.n_layers - 1: output = hid_dp(output)
 self.hidden = to_detach(new_hidden, cpu=False, gather=False)
 return output
def _change_hidden(self, bs):
self.hidden = [self._change_one_hidden(l, bs)for l in range(self.n_layers)]
self.bs = bs
def _one_rnn(self, n_in, n_out, bidir, weight_p, l):
"Return one of the inner rnn"
rnn = nn.LSTM(n_in, n_out, 1, batch_first=True, bidirectional=bidir, bias=False)
 return WeightDropout(rnn, weight_p)
 def _one_hidden(self, l):
"Return one hidden state"
nh =(self.n_hid)// self.n_dir
return(one_param(self ).new_zeros(self.n_dir, self.bs, nh ).to(device), one_param(self ).new_zeros(self.n_dir, self.bs, nh ).to(device))
def _change_one_hidden(self, l, bs):
if self.bs < bs:
nh =(self.n_hid)// self.n_dir
return tuple(torch.cat([h, h.new_zeros(self.n_dir, bs-self.bs, nh)], dim=1)for h in self.hidden[l])
 if self.bs > bs: return(self.hidden[l][0][:,:bs].contiguous() , self.hidden[l][1][:,:bs].contiguous())
 return self.hidden[l]
def reset(self):
"Reset the hidden states"
[r.reset() for r in self.rnns if hasattr(r, 'reset')]
self.hidden = [self._one_hidden(l)for l in range(self.n_layers)]
list_of_cast_genders = list(df['cast'].apply(lambda x: [i['gender'] for i in x] if x != {} else [] ).values)
 df['genders_0_cast'] = df['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
df['genders_1_cast'] = df['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
df['genders_2_cast'] = df['cast'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
df[['genders_0_cast', 'genders_1_cast']] = df[['genders_0_cast', 'genders_1_cast']].fillna(df[['genders_0_cast', 'genders_1_cast']].mean() )<feature_engineering>
lstm_width = 512
layers = [lstm_width * 3] + [lstm_width] + [9]
class MultiHeadedSequenceClassifier(Module):
"dim: input sequence feature dim"
def __init__(self, bptt=72, input_pad_idx=input_pad_idx, n_meta=1, dim=1024):
store_attr('input_pad_idx')
 self.awd_lstm = AWD_LSTM(dim+n_meta, lstm_width, 2, bidir=True)
 self.encoder = SentenceEncoder(bptt=bptt, module=self.awd_lstm, pad_idx=input_pad_idx)
 self.seq_head = LinearDecoder(1, lstm_width, bias=True)
 self.exam_head = PoolingLinearClassifier(layers, ps=[0.4, 0.1], bptt=bptt)
 def forward(self, x):
out, mask = self.encoder(x)
 seq_cls_out,_,_ = self.seq_head(out)
 seq_cls_out = seq_cls_out.squeeze(-1)
 exam_out,_,_ = self.exam_head(( out,mask))
return(seq_cls_out, exam_out)
 def predict(self, x):
out = self.awd_lstm(x, from_embeds=True)
 seq_cls_out,_,_ = self.seq_head(out)
 seq_cls_out = seq_cls_out.squeeze(-1)
 mask = torch.zeros(x.shape[:-1] ).bool().to(device)
 exam_out,_,_ = self.exam_head(( out, mask))
return(seq_cls_out, exam_out)
df['num_crew'] = df['crew'].apply(len )<sort_values>
layers1 = [512 * 3] + [512] + [12]
class MultiHeadedSoftmaxSequenceClassifier(Module):
"dim: input sequence feature dim"
def __init__(self, bptt=72, input_pad_idx=input_pad_idx, n_meta=2, dim=1024):
store_attr('input_pad_idx')
 self.awd_lstm = AWD_LSTM(dim+n_meta, 512, 2, bidir=True)
 self.encoder = SentenceEncoder(bptt=bptt, module=self.awd_lstm, pad_idx=input_pad_idx)
 self.seq_head = LinearDecoder(1, 512, bias=True)
 self.exam_head = PoolingLinearClassifier(layers1, ps=[0.4, 0.1], bptt=bptt)
 def forward(self, x):
out, mask = self.encoder(x)
 seq_cls_out,_,_ = self.seq_head(out)
 seq_cls_out = seq_cls_out.squeeze(-1)
 exam_out,_,_ = self.exam_head(( out,mask))
posneg_out, rvlv_out, lrc_out, chroacute_out =(exam_out[:,:3],
exam_out[:,3:6],
exam_out[:,6:9],
exam_out[:,9:])
 return(seq_cls_out, posneg_out, rvlv_out, lrc_out, chroacute_out)
 def predict(self, x):
out = self.awd_lstm(x, from_embeds=True)
 seq_cls_out,_,_ = self.seq_head(out)
 seq_cls_out = seq_cls_out.squeeze(-1)
 mask = torch.zeros(x.shape[:-1] ).bool().to(device)
 exam_out,_,_ = self.exam_head(( out, mask))
posneg_out, rvlv_out, lrc_out, chroacute_out =(exam_out[:,:3],
exam_out[:,3:6],
exam_out[:,6:9],
exam_out[:,9:])
 return(seq_cls_out, posneg_out, rvlv_out, lrc_out, chroacute_out )
department_count = pd.Series(Counter([job for lst in df["crew"].apply(lambda x : [ i["department"] for i in x] ).values for job in lst]))
department_count.sort_values(ascending=False )<create_dataframe>
seq_model0 = SequentialRNN(MultiHeadedSoftmaxSequenceClassifier(bptt=256, dim=1024, n_meta=2))
seq_model0.load_state_dict(torch.load(seqmodelpath/"nometa_sequence_softmax_with_preds_fulldata_fold0.pth")) ;
seq_model1 = SequentialRNN(MultiHeadedSoftmaxSequenceClassifier(bptt=256, dim=1024, n_meta=2))
seq_model1.load_state_dict(torch.load(seqmodelpath/"nometa_sequence_softmax_with_preds_fulldata_fold1.pth")) ;
seq_model2 = SequentialRNN(MultiHeadedSoftmaxSequenceClassifier(bptt=256, dim=1024, n_meta=2))
seq_model2.load_state_dict(torch.load(seqmodelpath/"nometa_sequence_softmax_with_preds_fulldata_fold2.pth")) ;
seq_model3 = SequentialRNN(MultiHeadedSoftmaxSequenceClassifier(bptt=256, dim=1024, n_meta=2))
seq_model3.load_state_dict(torch.load(seqmodelpath/"nometa_sequence_softmax_with_preds_fulldata_fold3.pth")) ;
seq_model4 = SequentialRNN(MultiHeadedSoftmaxSequenceClassifier(bptt=256, dim=1024, n_meta=2))
seq_model4.load_state_dict(torch.load(seqmodelpath/"nometa_sequence_softmax_with_preds_fulldata_fold4.pth")) ;
df_crew = { idx : pd.DataFrame([ [crew["department"], crew["job"], crew["name"]]
for crew in x], columns=["department", "job", "name"])
 for idx, x in df["crew"].iteritems() }<concatenate>
seq_model0 = seq_model0[0].to(device ).eval()
seq_model1 = seq_model1[0].to(device ).eval()
seq_model2 = seq_model2[0].to(device ).eval()
seq_model3 = seq_model3[0].to(device ).eval()
seq_model4 = seq_model4[0].to(device ).eval()
df_crew = pd.concat(df_crew)
 df_crew.head()<feature_engineering>
mean_std_dict = {
'scaled_position': [0.5078721739409284, 0.29139548181397823]
}
def select_job(list_dict, key, value):
return [ dic["name"] for dic in list_dict if dic[key]==value]<feature_engineering>
from fastai.medical.imaging import *
import pydicom
for department in department_count.index:
df['dep_{}_num'.format(department)] = df["crew"].apply(select_job, key="department", value=department ).apply(len )<feature_engineering>
class EmbeddingHook:
def __init__(self, m):
self.embeddings, self.m = tensor([] ).to(device), m
if len(m._forward_hooks)> 0: self.reset()
self.hook = Hook(m, self.hook_fn, cpu=False)
 def hook_fn(self, m, inp, out):
"Stack and save computed embeddings"
self.embeddings = torch.cat([self.embeddings, out])
 def reset(self):
self.m._forward_hooks = OrderedDict()
df_crewname = pd.DataFrame([], index=df.index)
 for job in ["Producer", "Director", "Screenplay", "Casting", "Original Music Composer","Writer"]:
col = 'job_{}_list'.format(job)
 df[col] = df["crew"].apply(select_job, key="job", value=job)
 top_list = [m[0] for m in Counter([i for j in df[col] for i in j] ).most_common(15)]
for i in top_list:
df_crewname['{}_{}'.format(job,i)] = df[col].apply(lambda x: i in x )<feature_engineering>
meta_feat_cols = ['scaled_position']
for job in ["Sound", "Art", "Costume & Make-Up", "Camera", "Visual Effects"]:
col = 'department_{}_list'.format(job)
 df[col] = df["crew"].apply(select_job, key="department", value=job)
 top_list = [m[0] for m in Counter([i for j in df[col] for i in j] ).most_common(15)]
for i in top_list:
df_crewname['{}_{}'.format(job,i)] = df[col].apply(lambda x: i in x )<feature_engineering>
def minmax_scaler(o): return(o - min(o)) /(max(o)- min(o))
df['job_Animation_num'] = df["crew"].apply(select_job, key="job", value="Animation" ).apply(len )<feature_engineering>
def read_dcm_img_v2(dcm, windows=windows):
"Read single slice in RGB"
return [dcm.windowed(*w)for w in windows]
df['genders_0_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
df['genders_1_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
df['genders_2_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
df[['genders_0_crew', 'genders_1_crew','genders_2_crew']] = df[['genders_0_crew', 'genders_1_crew','genders_2_crew']].fillna(df[['genders_0_crew', 'genders_1_crew','genders_2_crew']].mean() )<feature_engineering>
def predict_study(study_dirname):
study_df = test_df.query(f'StudyInstanceUID == "{study_dirname.stem}"')
 sop_ids = study_df['SOPInstanceUID'].values
study_files = str(testdatapath)+ "/" + study_df['StudyInstanceUID'] + "/" + study_df['SeriesInstanceUID'] + '/' + sop_ids + '.dcm'
dcm_ds = array([pydicom.read_file(o)for o in study_files])
 z_positions = array([int(o.ImagePositionPatient[-1])for o in dcm_ds])
 sortidxs = np.argsort(z_positions)
 sop_ids = sop_ids[sortidxs]
dcm_ds = dcm_ds[sortidxs]
z_positions = z_positions[sortidxs]
imgs = []
for o in dcm_ds: imgs += read_dcm_img_v2(o)
 h,w = imgs[0].size()
imgs = torch.stack(imgs ).view(-1,3,h,w)
 meta_embeddings = tensor(( minmax_scaler(z_positions)- mean)/std ).unsqueeze(1)
 emb_hook0 = EmbeddingHook(model0[1][1])
 emb_hook1 = EmbeddingHook(model1[1][1])
 emb_hook2 = EmbeddingHook(model2[1][1])
 emb_hook3 = EmbeddingHook(model3[1][1])
 emb_hook4 = EmbeddingHook(model4[1][1])
 with torch.no_grad() :
test_dl = learn2d_fold0.dls.test_dl(imgs.numpy() , bs=32, num_workers=2)
 outs0, outs1, outs2, outs3, outs4, outs5 = [],[],[],[],[],[]
for xb in test_dl:
out0 = model0(*xb); outs0.append(out0)
 out1 = model1(*xb); outs1.append(out1)
 out2 = model2(*xb); outs2.append(out2)
 out3 = model3(*xb); outs3.append(out3)
 out4 = model4(*xb); outs4.append(out4)
 outs0 = torch.cat(outs0)[:,1].view(-1,1)
 outs1 = torch.cat(outs1)[:,1].view(-1,1)
 outs2 = torch.cat(outs2)[:,1].view(-1,1)
 outs3 = torch.cat(outs3)[:,1].view(-1,1)
 outs4 = torch.cat(outs4)[:,1].view(-1,1)
 seq_inp0 = torch.cat([emb_hook0.embeddings, meta_embeddings.to(device), outs0],1)
 seq_inp1 = torch.cat([emb_hook1.embeddings, meta_embeddings.to(device), outs1],1)
 seq_inp2 = torch.cat([emb_hook2.embeddings, meta_embeddings.to(device), outs2],1)
 seq_inp3 = torch.cat([emb_hook3.embeddings, meta_embeddings.to(device), outs3],1)
 seq_inp4 = torch.cat([emb_hook4.embeddings, meta_embeddings.to(device), outs4],1)
 seq_img_preds0, posneg_preds0, rvlv_preds0, lrc_preds0, chroacute_preds0 = seq_model0.predict(seq_inp0[None,...])
 seq_img_preds1, posneg_preds1, rvlv_preds1, lrc_preds1, chroacute_preds1 = seq_model1.predict(seq_inp1[None,...])
 seq_img_preds2, posneg_preds2, rvlv_preds2, lrc_preds2, chroacute_preds2 = seq_model2.predict(seq_inp2[None,...])
 seq_img_preds3, posneg_preds3, rvlv_preds3, lrc_preds3, chroacute_preds3 = seq_model3.predict(seq_inp3[None,...])
 seq_img_preds4, posneg_preds4, rvlv_preds4, lrc_preds4, chroacute_preds4 = seq_model4.predict(seq_inp4[None,...])
 seq_img_preds0_tta, posneg_preds0_tta, rvlv_preds0_tta, lrc_preds0_tta, chroacute_preds0_tta = seq_model0.predict(seq_inp0.flip(dims=[0])[None,...])
 seq_img_preds1_tta, posneg_preds1_tta, rvlv_preds1_tta, lrc_preds1_tta, chroacute_preds1_tta = seq_model1.predict(seq_inp1.flip(dims=[0])[None,...])
 seq_img_preds2_tta, posneg_preds2_tta, rvlv_preds2_tta, lrc_preds2_tta, chroacute_preds2_tta = seq_model2.predict(seq_inp2.flip(dims=[0])[None,...])
 seq_img_preds3_tta, posneg_preds3_tta, rvlv_preds3_tta, lrc_preds3_tta, chroacute_preds3_tta = seq_model3.predict(seq_inp3.flip(dims=[0])[None,...])
 seq_img_preds4_tta, posneg_preds4_tta, rvlv_preds4_tta, lrc_preds4_tta, chroacute_preds4_tta = seq_model4.predict(seq_inp4.flip(dims=[0])[None,...])
 seq_img_preds0_tta = seq_img_preds0_tta.flip(dims=[1])
 seq_img_preds1_tta = seq_img_preds1_tta.flip(dims=[1])
 seq_img_preds2_tta = seq_img_preds2_tta.flip(dims=[1])
 seq_img_preds3_tta = seq_img_preds3_tta.flip(dims=[1])
 seq_img_preds4_tta = seq_img_preds4_tta.flip(dims=[1])
 seq_img_preds0,posneg_preds0,rvlv_preds0,lrc_preds0,chroacute_preds0 =(( seq_img_preds0+seq_img_preds0_tta)/2, 
(posneg_preds0+posneg_preds0_tta)/2, 
(rvlv_preds0+rvlv_preds0_tta)/2, 
(lrc_preds0+lrc_preds0_tta)/2, 
(chroacute_preds0+chroacute_preds0_tta)/2)
 seq_img_preds1,posneg_preds1,rvlv_preds1,lrc_preds1,chroacute_preds1 =(( seq_img_preds1+seq_img_preds1_tta)/2, 
(posneg_preds1+posneg_preds1_tta)/2, 
(rvlv_preds1+rvlv_preds1_tta)/2, 
(lrc_preds1+lrc_preds1_tta)/2, 
(chroacute_preds1+chroacute_preds1_tta)/2)
 seq_img_preds2,posneg_preds2,rvlv_preds2,lrc_preds2,chroacute_preds2 =(( seq_img_preds2+seq_img_preds2_tta)/2, 
(posneg_preds2+posneg_preds2_tta)/2, 
(rvlv_preds2+rvlv_preds2_tta)/2, 
(lrc_preds2+lrc_preds2_tta)/2, 
(chroacute_preds2+chroacute_preds2_tta)/2)
 seq_img_preds3,posneg_preds3,rvlv_preds3,lrc_preds3,chroacute_preds3 =(( seq_img_preds3+seq_img_preds3_tta)/2, 
(posneg_preds3+posneg_preds3_tta)/2, 
(rvlv_preds3+rvlv_preds3_tta)/2, 
(lrc_preds3+lrc_preds3_tta)/2, 
(chroacute_preds3+chroacute_preds3_tta)/2)
 seq_img_preds4,posneg_preds4,rvlv_preds4,lrc_preds4,chroacute_preds4 =(( seq_img_preds4+seq_img_preds4_tta)/2, 
(posneg_preds4+posneg_preds4_tta)/2, 
(rvlv_preds4+rvlv_preds4_tta)/2, 
(lrc_preds4+lrc_preds4_tta)/2, 
(chroacute_preds4+chroacute_preds4_tta)/2)
 seq_img_preds0 = seq_img_preds0.sigmoid()
seq_img_preds1 = seq_img_preds1.sigmoid()
seq_img_preds2 = seq_img_preds2.sigmoid()
seq_img_preds3 = seq_img_preds3.sigmoid()
seq_img_preds4 = seq_img_preds4.sigmoid()
posneg_preds =((
posneg_preds0.softmax(1)
 + posneg_preds1.softmax(1)
 + posneg_preds2.softmax(1)
 + posneg_preds3.softmax(1)
 + posneg_preds4.softmax(1)) / 5)[0]
rvlv_preds =((
rvlv_preds0.softmax(1)
 + rvlv_preds1.softmax(1)
 + rvlv_preds2.softmax(1)
 + rvlv_preds3.softmax(1)
 + rvlv_preds4.softmax(1)) / 5)[0]
lrc_preds =((
lrc_preds0.sigmoid()
+ lrc_preds1.sigmoid()
+ lrc_preds2.sigmoid()
+ lrc_preds3.sigmoid()
+ lrc_preds4.sigmoid())/ 5)[0]
chroacute_preds =((
chroacute_preds0.softmax(1)
 + chroacute_preds1.softmax(1)
 + chroacute_preds2.softmax(1)
 + chroacute_preds3.softmax(1)
 + chroacute_preds4.softmax(1)) / 5)[0]
_, negative, indeterminate = posneg_preds
rvlv_gte, rvlv_lt, _ = rvlv_preds
left_pe, right_pe, central_pe = lrc_preds
chro_pe, acute_chro_pe, _ = chroacute_preds
seq_exam_preds = torch.stack([negative, rvlv_gte, rvlv_lt, left_pe, chro_pe, right_pe, acute_chro_pe, central_pe, indeterminate])
 seq_img_preds =(( seq_img_preds0
+ seq_img_preds1
+ seq_img_preds2
+ seq_img_preds3
+ seq_img_preds4)/ 5)[0]
return(sop_ids, to_cpu(seq_img_preds), to_cpu(seq_exam_preds))
df['budget_runtime_ratio'] = df['budget']/df['runtime']<feature_engineering>
def get_study_res(sid, sop_ids, img_preds, exam_preds):
"Get preds from 2D and 3D cnn models"
sub_res = []
for sopid, p in zip(sop_ids, to_np(img_preds)) :
sub_res.append(( sopid, p))
target_cols = [
'negative_exam_for_pe',
'rv_lv_ratio_gte_1',
'rv_lv_ratio_lt_1',
'leftsided_pe',
'chronic_pe',
'rightsided_pe',
'acute_and_chronic_pe',
'central_pe',
'indeterminate'
]
for tcol, p in zip(target_cols, to_np(exam_preds)) :
sub_res.append(( f"{sid}_{tcol}", p))
return sub_res
df['budget_popularity_ratio'] = df['budget']/df['popularity']<filter>
%%time
do_full = False
n = 20
if Path('.. /input/rsna-str-pulmonary-embolism-detection/train' ).exists() and not do_full:
test_study_dirnames = [datapath/'test'/o for o in test_df['StudyInstanceUID'].unique() ]
test_study_dirnames = np.random.choice(test_study_dirnames, n, replace=False)
 sub_res = []
for study_dirname in test_study_dirnames:
sop_ids, seq_img_preds, seq_exam_preds = predict_study(study_dirname)
 study_res = get_study_res(study_dirname.stem, sop_ids, seq_img_preds, seq_exam_preds)
 sub_res += study_res
rating_na = df[df["rating"].isnull() ]<feature_engineering>
final_sub_df = pd.DataFrame(sub_res, columns=['id', 'label'])
 final_sub_df['label'] = np.clip(final_sub_df['label'], 0.0001, 0.9999 )
df['budget_popularity2_ratio'] = df['budget']/df['popularity2']<feature_engineering>
final_sub_df.to_csv("submission.csv", index=False )
df['budget_year_ratio'] = df['budget']/df['release_year']<feature_engineering>
def check_consistency(sub, test):

 df_tmp = sub.loc[sub.id.str.contains('_', regex = False)].reset_index(drop = True)
 df_tmp['StudyInstanceUID'] = df_tmp['id'].apply(lambda x:str(x ).split('_')[0])
 df_tmp['label_type'] = df_tmp['id'].apply(lambda x:'_'.join(str(x ).split('_')[1:]))
df_exam = df_tmp.pivot(index = 'StudyInstanceUID', columns = 'label_type', values = 'label')
 df_image = sub.loc[sub.id.isin(test.SOPInstanceUID)].reset_index(drop = True)
 df_image = df_image.merge(test, how = 'left', left_on = 'id', right_on = 'SOPInstanceUID')
 df_image.rename(columns = {"label": "pe_present_on_image"}, inplace = True)
 del df_image['id']
df = df_exam.merge(df_image, how = 'left', on = 'StudyInstanceUID')
 ids = ['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID']
labels = [c for c in df.columns if c not in ids]
df = df[ids + labels]
df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID'] ).pe_present_on_image.max())
 df_pos = df.loc[df.positive_images_in_exam > 0.5]
df_neg = df.loc[df.positive_images_in_exam <= 0.5]
rule1a = df_pos.loc[(( df_pos.rv_lv_ratio_lt_1 > 0.5)& 
(df_pos.rv_lv_ratio_gte_1 > 0.5)) | 
(( df_pos.rv_lv_ratio_lt_1 <= 0.5)& 
(df_pos.rv_lv_ratio_gte_1 <= 0.5)) ].reset_index(drop = True)
 rule1a['broken_rule'] = '1a'
rule1b = df_pos.loc[(df_pos.central_pe <= 0.5)& 
(df_pos.rightsided_pe <= 0.5)& 
(df_pos.leftsided_pe <= 0.5)].reset_index(drop = True)
 rule1b['broken_rule'] = '1b'
rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5)& 
(df_pos.chronic_pe > 0.5)].reset_index(drop = True)
 rule1c['broken_rule'] = '1c'
rule1d = df_pos.loc[(df_pos.indeterminate > 0.5)| 
(df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)
 rule1d['broken_rule'] = '1d'
rule2a = df_neg.loc[(( df_neg.indeterminate > 0.5)& 
(df_neg.negative_exam_for_pe > 0.5)) | 
(( df_neg.indeterminate <= 0.5)& 
(df_neg.negative_exam_for_pe <= 0.5)) ].reset_index(drop = True)
 rule2a['broken_rule'] = '2a'
rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1 > 0.5)| 
(df_neg.rv_lv_ratio_gte_1 > 0.5)| 
(df_neg.central_pe > 0.5)| 
(df_neg.rightsided_pe > 0.5)| 
(df_neg.leftsided_pe > 0.5)| 
(df_neg.acute_and_chronic_pe > 0.5)| 
(df_neg.chronic_pe > 0.5)].reset_index(drop = True)
 rule2b['broken_rule'] = '2b'
errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)
 print('Found', len(errors), 'inconsistent predictions')
 if len(errors)> 0:
print(errors.broken_rule.value_counts())
 return errors
df['production_countries_count'] = df['production_countries'].apply(lambda x : len(x))
df['production_companies_count'] = df['production_companies'].apply(lambda x : len(x))<data_type_conversions>
%%time
consistency_df0 = check_consistency(final_sub_df, test_df )
df["collection_name"] = df["belongs_to_collection"].apply(lambda x : x[0]["name"] if len(x)>0 else 0)
 le.fit(list(df['collection_name'].fillna('')))
 df['collection_name'] = le.transform(df['collection_name'].fillna('' ).astype(str))<categorify>
consistency_df0['broken_rule'].value_counts()
df['mean_pop1_bud'] = df.groupby('popularity')['budget'].transform('mean')
 df['mean_pop2_bud'] = df.groupby('popularity2')['budget'].transform('mean')
 df['mean_year_bud'] = df.groupby('release_year')['budget'].transform('mean')
 df['mean_pop1_rate'] = df.groupby('popularity')['rating'].transform('mean')
 df['mean_pop2_rate'] = df.groupby('popularity2')['rating'].transform('mean')
 df['mean_rate_tV'] = df.groupby('rating')['totalVotes'].transform('mean' )<categorify>
from scipy.special import softmax
df['runtime_to_mean_year'] = df['runtime'] / df.groupby("release_year")["runtime"].transform('mean')
 df['popularity_to_mean_year'] = df['popularity'] / df.groupby("release_year")["popularity"].transform('mean')
 df['budget_to_mean_year'] = df['budget'] / df.groupby("release_year")["budget"].transform('mean' )<concatenate>
def solve_rule_1b(sub, errors, log=False):
studies_break_rule = errors.query("(central_pe <= 0.5)&(rightsided_pe <= 0.5)&(leftsided_pe <= 0.5)")['StudyInstanceUID'].unique()
if log:
print(studies_break_rule,studies_break_rule.shape)
 for i in studies_break_rule:
idx = sub[sub['id'].str.contains(i)].index.values
old = sub.loc[idx[[3,5,7]], 'label'].values
best = np.argmax(old)
 new = old.copy()
if(old[0] == 0.5)and(old[1] == 0.5)and(old[2] == 0.5):
pass
else:
new[best] = new[best] + 0.5
new = np.clip(new , 0, 0.51)
 if log:
print(old, new)
 sub.loc[idx[[3,5,7]], 'label'] = new
df_features = pd.concat(dfdic_feature, axis=1 )<feature_engineering>
def solve_rule_1c(sub, errors, log=False):
studies_break_rule = errors.query("(acute_and_chronic_pe > 0.5)&(chronic_pe > 0.5)")['StudyInstanceUID'].unique()
if log:
print(studies_break_rule, studies_break_rule.shape)
 for i in studies_break_rule:
idx = sub[sub['id'].str.contains(i)].index.values
old = sub.loc[idx[[4,6]], 'label'].values
new = softmax(old)
 if log:
print(old , new)
 sub.loc[idx[[4,6]], 'label'] == new
df_features.index = df.index<drop_column>
def solve_rule_1d(sub, errors, log=False):
studies_break_rule = errors.query("(indeterminate > 0.5)|(negative_exam_for_pe > 0.5)")['StudyInstanceUID'].unique()
if log:
print(studies_break_rule, studies_break_rule.shape)
 for i in studies_break_rule:
idx = sub[sub['id'].str.contains(i)].index.values
old = sub.loc[idx[[0,-1]], 'label'].values
new = np.clip(old, 0., 0.4)
 if log:
print(old , new)
 sub.loc[idx[[0,-1]], 'label'] = new
df_use = df[['num_cast', 'genders_0_cast','runtime_to_mean_year','budget_to_mean_year',"log_budget",
'genders_1_cast','genders_2_cast', 'num_crew', 'genders_0_crew', 'genders_1_crew','genders_2_crew',
"tagline_word_count","overview_word_count","title_word_count","has_homepage",
'popularity','runtime','release_year', 'release_month','release_dayofweek',"num_genres"
,"popularity2","rating","totalVotes",'isOriginalLanguageEng',
'budget_runtime_ratio','budget_popularity_ratio','budget_year_ratio','budget_popularity2_ratio',
'production_countries_count','production_companies_count','mean_pop1_bud','mean_pop2_bud','mean_year_bud','mean_pop1_rate',
'mean_pop2_rate','mean_rate_tV',
'dep_Directing_num', 'dep_Writing_num', 'dep_Production_num',
'dep_Sound_num', 'dep_Camera_num', 'dep_Editing_num', 'dep_Art_num',
'dep_Costume & Make-Up_num', 'dep_Crew_num', 'dep_Lighting_num',
'dep_Visual Effects_num', 'dep_Actors_num', 'job_Animation_num' ]]<concatenate>
def solve_negative(sub, errors):
negative_sopids = errors.query("negative_exam_for_pe > 0.5")['SOPInstanceUID'].values
neg_clip_idxs = sub[sub['id'].isin(negative_sopids)].index
sub.loc[neg_clip_idxs, 'label'] = np.clip(sub.loc[neg_clip_idxs, 'label'], 0, 0.5 )
df_use = pd.concat([df_use, df_features], axis=1 )<rename_columns>
%%time
solve_negative(sub=final_sub_df, errors=consistency_df0 )
df_use.columns = ["".join(c if c.isalnum() else "_" for c in str(x)) for x in df_use.columns]<count_missing_values>
%%time
consistency_df0 = check_consistency(sub=final_sub_df, test=test_df )
df_use.isnull().sum().sum()<prepare_x_and_y>
%%time
solve_rule_1a(sub=final_sub_df, errors=consistency_df0, log=False)
 solve_rule_1b(sub=final_sub_df, errors=consistency_df0, log=False)
 solve_rule_1c(sub=final_sub_df, errors=consistency_df0, log=False)
 solve_rule_1d(sub=final_sub_df, errors=consistency_df0, log=False )
trainX = df_use.iloc[:train.shape[0],:].reset_index(drop=True)
 test_X = df_use.iloc[train.shape[0]:,:].reset_index(drop=True)
 trainy = np.log1p(train["revenue"] )<split>
consistency_df_final = check_consistency(final_sub_df, test_df )
X_train, X_test, y_train, y_test = train_test_split(trainX,trainy,test_size=0.3,random_state=100 )<choose_model_class>
submit_fixed = True
if submit_fixed:
final_sub_df.to_csv("submission.csv", index=False )
<train_model><EOS> 
final_sub_df['label_prefix'] = final_sub_df.apply(lambda o: "_".join(o['id'].split("_")[1:]), 1)
 stats = final_sub_df.groupby("label_prefix" ).agg(['min', 'max', 'mean', 'median'])
 stats