Kaggle competition: RSNA STR Pulmonary Embolism Detection
<SOS> metric: weightedmeancolumnwiselogloss Kaggle data source: rsna-str-pulmonary-embolism-detection<feature_engineering>
!cp /kaggle/input/gdcm-conda-install/gdcm.tar.
 !tar -xzf gdcm.tar
!conda install -q --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2
%%time
dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
for col in dict_columns:
df[col]=df[col].apply(lambda x: [] if pd.isna(x)else ast.literal_eval(x))<categorify>
from fastai.vision.all import *
from fastai.medical.imaging import *
def count_word_list(series):
len_max = series.apply(len ).max()
tmp = series.map(lambda x: x+["nashi"]*(len_max-len(x)))
 word_set = set(sum(list(series.values), []))
for n in range(len_max):
word_dfn = pd.get_dummies(tmp.apply(lambda x: x[n]))
word_dfn = word_dfn.reindex(word_set, axis=1 ).fillna(0 ).astype(int)
 if n==0:
word_df = word_dfn
else:
word_df = word_df + word_dfn
return word_df<feature_engineering>
datapath = Path("/kaggle/input/rsna-str-pulmonary-embolism-detection/")
 testdatapath = datapath/'test'
cnn2dmodelpath = Path("/kaggle/input/rsnape2dmodels/")
 cnn3dmodelpath = Path("/kaggle/input/rsnape3dmodels/")
 seqmodelpath = Path("/kaggle/input/rsnapeseqmodels/")
 test_df = pd.read_csv(datapath/'test.csv')
 sub_df = pd.read_csv(datapath/'sample_submission.csv' )
df["genre_names"] = df["genres"].apply(lambda x : [ i["name"] for i in x])
<drop_column>
[o for o in sub_df['id'].values if "df06fad17bc3" in o]
dfdic_feature["genre"] = count_word_list(df["genre_names"])
 dfdic_feature["genre"] = dfdic_feature["genre"].drop("TV Movie", axis=1)
 dfdic_feature["genre"].head()<feature_engineering>
lung_window =(1500, -600)
 pe_window =(700, 100)
 mediastinal_window =(400, 40)
 windows =(lung_window, pe_window, mediastinal_window)
 def read_dcm_img(dcm, windows=windows):
"Read single slice in RGB"
return torch.stack([dcm.windowed(*w)for w in windows] )
n_language = df.loc[:train.index[-1], "original_language"].value_counts()
large_language = n_language[n_language>=10].index
df.loc[~df["original_language"].isin(large_language), "original_language"] = "small"<data_type_conversions>
def get_dls(tensors, size=256, bs=128):
"Get study dataloader"
tfms = [[RandomResizedCropGPU(size, min_scale=0.9)], []]
dsets = Datasets(tensors, tfms=tfms, splits=([0,1], [2,3]))
batch_tfms = []
dls = dsets.dataloaders(bs=bs, after_batch=batch_tfms, num_workers=2)
 return dls
dls = get_dls(torch.zeros(4, 3, 512, 512), size=480, bs=32)
 dls.c = 2
df["original_language"] = df["original_language"].astype("category" )<feature_engineering>
learn2d_fold0 = cnn_learner(dls, xresnet34, pretrained=False, loss_func=nn.CrossEntropyLoss() , model_dir='.')
 learn2d_fold0.path = cnn2dmodelpath
learn2d_fold0.load('xresnet34-512-PR-fold0');
<feature_engineering>
learn2d_fold2 = cnn_learner(dls, xresnet34, pretrained=False, loss_func=nn.CrossEntropyLoss() , model_dir='.')
 learn2d_fold2.path = cnn2dmodelpath
learn2d_fold2.load('xresnet34-512-PR-fold2');
df["production_names"] = df["production_companies"].apply(lambda x : [ i["name"] for i in x])
<define_variables>
learn2d_fold3 = cnn_learner(dls, xresnet34, pretrained=False, loss_func=nn.CrossEntropyLoss() , model_dir='.')
 learn2d_fold3.path = cnn2dmodelpath
learn2d_fold3.load('xresnet34-512-PR-fold3');
tmp = count_word_list(df["production_names"] )<sort_values>
learn2d_fold4 = cnn_learner(dls, xresnet34, pretrained=False, loss_func=nn.CrossEntropyLoss() , model_dir='.')
 learn2d_fold4.path = cnn2dmodelpath
learn2d_fold4.load('xresnet34-512-PR-fold4');
def select_top_n(df, topn=9999, nmin=2):
n_word =(df.loc[train["id"]]>0 ).sum().sort_values(ascending=False)
 smallmin = n_word[n_word<nmin].index
smalln = n_word.iloc[topn+1:].index
small = set(smallmin)| set(smalln)
 df["small"] = df[small].sum(axis=1)
 return df.drop(small, axis=1 )<feature_engineering>
model0 = learn2d_fold0.model.eval().to(device)
 model2 = learn2d_fold2.model.eval().to(device)
 model3 = learn2d_fold3.model.eval().to(device)
 model4 = learn2d_fold4.model.eval().to(device )
df["country_names"] = df["production_countries"].apply(lambda x : [ i["name"] for i in x])
 df_country = count_word_list(df["country_names"] )<data_type_conversions>
from fastai.text.all import *
df_country =(df_country.T/df_country.sum(axis=1)).T.fillna(0 )<feature_engineering>
input_pad_idx = None
<feature_engineering>
class AWD_LSTM(Module):
"AWD-LSTM inspired by https://arxiv.org/abs/1708.02182"
initrange=0.1
def __init__(self, emb_sz,n_hid, n_layers, hidden_p=0.2, input_p=0.6, weight_p=0.5, bidir=False):
store_attr('emb_sz,n_hid,n_layers')
 self.bs = 1
self.n_dir = 2 if bidir else 1
self.rnns = nn.ModuleList([self._one_rnn(emb_sz if l == 0 else n_hid,(n_hid)//self.n_dir, bidir, weight_p, l)for l in range(n_layers)])
 self.input_dp = RNNDropout(input_p)
 self.hidden_dps = nn.ModuleList([RNNDropout(hidden_p)for l in range(n_layers)])
 self.reset()
def forward(self, x, from_embeds=False):
if from_embeds: inp = x
else: inp = combined_embeddings[x].to(device)
 bs,sl = inp.shape[:2]
if bs!=self.bs: self._change_hidden(bs)
 output = self.input_dp(inp)
 new_hidden = []
for l,(rnn,hid_dp)in enumerate(zip(self.rnns, self.hidden_dps)) :
output, new_h = rnn(output, self.hidden[l])
 new_hidden.append(new_h)
 if l != self.n_layers - 1: output = hid_dp(output)
 self.hidden = to_detach(new_hidden, cpu=False, gather=False)
 return output
def _change_hidden(self, bs):
self.hidden = [self._change_one_hidden(l, bs)for l in range(self.n_layers)]
self.bs = bs
def _one_rnn(self, n_in, n_out, bidir, weight_p, l):
"Return one of the inner rnn"
rnn = nn.LSTM(n_in, n_out, 1, batch_first=True, bidirectional=bidir, bias=False)
 return WeightDropout(rnn, weight_p)
 def _one_hidden(self, l):
"Return one hidden state"
nh =(self.n_hid)// self.n_dir
return(one_param(self ).new_zeros(self.n_dir, self.bs, nh ).to(device), one_param(self ).new_zeros(self.n_dir, self.bs, nh ).to(device))
def _change_one_hidden(self, l, bs):
if self.bs < bs:
nh =(self.n_hid)// self.n_dir
return tuple(torch.cat([h, h.new_zeros(self.n_dir, bs-self.bs, nh)], dim=1)for h in self.hidden[l])
 if self.bs > bs: return(self.hidden[l][0][:,:bs].contiguous() , self.hidden[l][1][:,:bs].contiguous())
 return self.hidden[l]
def reset(self):
"Reset the hidden states"
[r.reset() for r in self.rnns if hasattr(r, 'reset')]
self.hidden = [self._one_hidden(l)for l in range(self.n_layers)]
df["keyword_list"] = df["Keywords"].apply(lambda x : [ i["name"] for i in x] )<feature_engineering>
lstm_width = 512
layers = [lstm_width * 3] + [lstm_width] + [9]
class MultiHeadedSequenceClassifier(Module):
"dim: input sequence feature dim"
def __init__(self, bptt=72, input_pad_idx=input_pad_idx, n_meta=1, dim=1024):
store_attr('input_pad_idx')
 self.awd_lstm = AWD_LSTM(dim+n_meta, lstm_width, 2, bidir=True)
 self.encoder = SentenceEncoder(bptt=bptt, module=self.awd_lstm, pad_idx=input_pad_idx)
 self.seq_head = LinearDecoder(1, lstm_width, bias=True)
 self.exam_head = PoolingLinearClassifier(layers, ps=[0.4, 0.1], bptt=bptt)
 def forward(self, x):
out, mask = self.encoder(x)
 seq_cls_out,_,_ = self.seq_head(out)
 seq_cls_out = seq_cls_out.squeeze(-1)
 exam_out,_,_ = self.exam_head(( out,mask))
return(seq_cls_out, exam_out)
 def predict(self, x):
out = self.awd_lstm(x, from_embeds=True)
 seq_cls_out,_,_ = self.seq_head(out)
 seq_cls_out = seq_cls_out.squeeze(-1)
 mask = torch.zeros(x.shape[:-1] ).bool().to(device)
 exam_out,_,_ = self.exam_head(( out, mask))
return(seq_cls_out, exam_out)
df["num_Keywords"] = df["keyword_list"].apply(len )<feature_engineering>
layers1 = [512 * 3] + [512] + [12]
class MultiHeadedSoftmaxSequenceClassifier(Module):
"dim: input sequence feature dim"
def __init__(self, bptt=72, input_pad_idx=input_pad_idx, n_meta=2, dim=1024):
store_attr('input_pad_idx')
 self.awd_lstm = AWD_LSTM(dim+n_meta, 512, 2, bidir=True)
 self.encoder = SentenceEncoder(bptt=bptt, module=self.awd_lstm, pad_idx=input_pad_idx)
 self.seq_head = LinearDecoder(1, 512, bias=True)
 self.exam_head = PoolingLinearClassifier(layers1, ps=[0.4, 0.1], bptt=bptt)
 def forward(self, x):
out, mask = self.encoder(x)
 seq_cls_out,_,_ = self.seq_head(out)
 seq_cls_out = seq_cls_out.squeeze(-1)
 exam_out,_,_ = self.exam_head(( out,mask))
posneg_out, rvlv_out, lrc_out, chroacute_out =(exam_out[:,:3],
exam_out[:,3:6],
exam_out[:,6:9],
exam_out[:,9:])
 return(seq_cls_out, posneg_out, rvlv_out, lrc_out, chroacute_out )
df_lang = pd.DataFrame(df['spoken_languages'])
 list_lang_names = list(df_lang['spoken_languages'].apply(lambda x: [i['name'] for i in x] if x != {} else [] ).values)
 df_lang['num_spoken_languages'] = df_lang['spoken_languages'].apply(lambda x: len(x)if x != {} else 0)
 df_lang['all_spoken_languages'] = df_lang['spoken_languages'].apply(lambda x: ' '.join(sorted([i['name'] for i in x])) if x != {} else '')
 top_lang_names = [m[0] for m in Counter([i for j in list_lang_names for i in j] ).most_common(15)]
for g in top_lang_names:
df_lang[g] = df_lang['all_spoken_languages'].apply(lambda x: 1 if g in x else 0 )<rename_columns>
seq_model0 = SequentialRNN(MultiHeadedSequenceClassifier(bptt=256, dim=1024, n_meta=2))
seq_model0.load_state_dict(torch.load(seqmodelpath/"nometa_sequence_with_preds_fulldata_fold0.pth")) ;
seq_model2 = SequentialRNN(MultiHeadedSequenceClassifier(bptt=256, dim=1024, n_meta=2))
seq_model2.load_state_dict(torch.load(seqmodelpath/"nometa_sequence_with_preds_fulldata_fold2.pth")) ;
seq_model3 = SequentialRNN(MultiHeadedSequenceClassifier(bptt=256, dim=1024, n_meta=2))
seq_model3.load_state_dict(torch.load(seqmodelpath/"nometa_sequence_with_preds_fulldata_fold3.pth")) ;
seq_model4 = SequentialRNN(MultiHeadedSequenceClassifier(bptt=256, dim=1024, n_meta=2))
seq_model4.load_state_dict(torch.load(seqmodelpath/"nometa_sequence_with_preds_fulldata_fold4.pth")) ;
df_lang.rename(columns ={'English':'spoken_en',
'Français':'spoken_fr',
'Español':'spoken_es',
'Deutsch':'spoken_gr',
'Pусский':'spoken_sv',
'Italiano':'spoken_it',
'日本語':'spoken_ja',
'普通话':'spoken_ch1',
'हिन्दी':'spoken_in',
'':'spoken_unknown',
'العربية':'spoken_arb',
'Português':'spoken_por',
'广州话 / 廣州話':'spoken_ch2',
'한국어/조선말':'spoken_kr',
'Polski':'spoken_pol'}, inplace =True )<drop_column>
seq_model0 = seq_model0[0].to(device ).eval()
seq_model2 = seq_model2[0].to(device ).eval()
seq_model3 = seq_model3[0].to(device ).eval()
seq_model4 = seq_model4[0].to(device ).eval()
df_lang.drop(columns=['spoken_languages', 'all_spoken_languages'], inplace = True )<feature_engineering>
mean_std_dict = {
'scaled_position': [0.5078721739409284, 0.29139548181397823]
}
df["language_names"] = df["spoken_languages"].apply(lambda x : [ i["name"] for i in x])
 df["n_language"] = df["language_names"].apply(len)
 df.loc[df["n_language"]==0, "n_language"] = 1<feature_engineering>
from fastai.medical.imaging import *
import pydicom
df["speak_English"] = df["language_names"].apply(lambda x : "English" in x )<categorify>
class EmbeddingHook:
def __init__(self, m):
self.embeddings, self.m = tensor([] ).to(device), m
if len(m._forward_hooks)> 0: self.reset()
self.hook = Hook(m, self.hook_fn, cpu=False)
 def hook_fn(self, m, inp, out):
"Stack and save computed embeddings"
self.embeddings = torch.cat([self.embeddings, out])
 def reset(self):
self.m._forward_hooks = OrderedDict()
df['speak_English'] = pd.get_dummies(df['speak_English'] )<feature_engineering>
meta_feat_cols = ['scaled_position']
df[['release_month','release_day','release_year']]=df['release_date'].str.split('/',expand=True ).replace(np.nan, -1 ).astype(int)
 df.loc[(df['release_year'] <= 19)&(df['release_year'] < 100), "release_year"] += 2000
df.loc[(df['release_year'] > 19)&(df['release_year'] < 100), "release_year"] += 1900
releaseDate = pd.to_datetime(df['release_date'])
 df['release_dayofweek'] = releaseDate.dt.dayofweek
df['release_quarter'] = releaseDate.dt.quarter<feature_engineering>
def minmax_scaler(o): return(o - min(o)) /(max(o)- min(o))
<feature_engineering>
def read_dcm_img_v2(dcm, windows=windows):
"Read single slice in RGB"
return [dcm.windowed(*w)for w in windows]
df["collection_name"] = df["belongs_to_collection"].apply(lambda x : x[0]["name"] if len(x)>0 else "nashi")
<merge>
def predict_study(study_dirname):
study_df = test_df.query(f'StudyInstanceUID == "{study_dirname.stem}"')
 sop_ids = study_df['SOPInstanceUID'].values
study_files = str(testdatapath)+ "/" + study_df['StudyInstanceUID'] + "/" + study_df['SeriesInstanceUID'] + '/' + sop_ids + '.dcm'
dcm_ds = array([pydicom.read_file(o)for o in study_files])
 z_positions = array([int(o.ImagePositionPatient[-1])for o in dcm_ds])
 sortidxs = np.argsort(z_positions)
 sop_ids = sop_ids[sortidxs]
dcm_ds = dcm_ds[sortidxs]
z_positions = z_positions[sortidxs]
imgs = []
for o in dcm_ds: imgs += read_dcm_img_v2(o)
 h,w = imgs[0].size()
imgs = torch.stack(imgs ).view(-1,3,h,w)
 meta_embeddings = tensor(( minmax_scaler(z_positions)- mean)/std ).unsqueeze(1)
 emb_hook0 = EmbeddingHook(model0[1][1])
 emb_hook2 = EmbeddingHook(model2[1][1])
 emb_hook3 = EmbeddingHook(model3[1][1])
 emb_hook4 = EmbeddingHook(model4[1][1])
 with torch.no_grad() :
test_dl = learn2d_fold0.dls.test_dl(imgs.numpy() , bs=32, num_workers=2)
 outs0, outs1, outs2, outs3, outs4, outs5 = [],[],[],[],[],[]
for xb in test_dl:
out0 = model0(*xb); outs0.append(out0)
 out2 = model2(*xb); outs2.append(out2)
 out3 = model3(*xb); outs3.append(out3)
 out4 = model4(*xb); outs4.append(out4)
 outs0 = torch.cat(outs0)[:,1].view(-1,1)
 outs2 = torch.cat(outs2)[:,1].view(-1,1)
 outs3 = torch.cat(outs3)[:,1].view(-1,1)
 outs4 = torch.cat(outs4)[:,1].view(-1,1)
 seq_inp0 = torch.cat([emb_hook0.embeddings, meta_embeddings.to(device), outs0],1)
 seq_inp2 = torch.cat([emb_hook2.embeddings, meta_embeddings.to(device), outs2],1)
 seq_inp3 = torch.cat([emb_hook3.embeddings, meta_embeddings.to(device), outs3],1)
 seq_inp4 = torch.cat([emb_hook4.embeddings, meta_embeddings.to(device), outs4],1)
 seq_img_preds0, seq_exam_preds0 = seq_model0.predict(seq_inp0[None,...])
 seq_img_preds2, seq_exam_preds2 = seq_model2.predict(seq_inp2[None,...])
 seq_img_preds3, seq_exam_preds3 = seq_model3.predict(seq_inp3[None,...])
 seq_img_preds4, seq_exam_preds4 = seq_model4.predict(seq_inp4[None,...])
 seq_img_preds0, seq_exam_preds0 = seq_img_preds0.sigmoid() , seq_exam_preds0.sigmoid()
seq_img_preds2, seq_exam_preds2 = seq_img_preds2.sigmoid() , seq_exam_preds2.sigmoid()
seq_img_preds3, seq_exam_preds3 = seq_img_preds3.sigmoid() , seq_exam_preds3.sigmoid()
seq_img_preds4, seq_exam_preds4 = seq_img_preds4.sigmoid() , seq_exam_preds4.sigmoid()
fold_wgts = array([1,0,1,1,1])
 seq_img_preds = to_cpu(( seq_img_preds0 *(fold_wgts[0])+
seq_img_preds2 *(fold_wgts[2])+
seq_img_preds3 *(fold_wgts[3])+
seq_img_preds4 *(fold_wgts[4])
)/ sum(fold_wgts)) [0]
seq_exam_preds = to_cpu(( seq_exam_preds0 *(fold_wgts[0])+
seq_exam_preds2 *(fold_wgts[2])+
seq_exam_preds3 *(fold_wgts[3])+
seq_exam_preds4 *(fold_wgts[4])
)/ sum(fold_wgts)) [0]
return(sop_ids, seq_img_preds, seq_exam_preds )
df["count_collection"] = df["collection_name"].apply(lambda x :(df["collection_name"]==x ).sum())
 df.loc[df["collection_name"]=="nashi", "count_collection"] = 0 
<feature_engineering>
def get_study_res(sid, sop_ids, img_preds, exam_preds):
"Get preds from 2D and 3D cnn models"
sub_res = []
for sopid, p in zip(sop_ids, to_np(img_preds)) :
sub_res.append(( sopid, p))
target_cols = [
'negative_exam_for_pe',
'rv_lv_ratio_gte_1',
'rv_lv_ratio_lt_1',
'leftsided_pe',
'chronic_pe',
'rightsided_pe',
'acute_and_chronic_pe',
'central_pe',
'indeterminate'
]
for tcol, p in zip(target_cols, to_np(exam_preds)) :
sub_res.append(( f"{sid}_{tcol}", p))
return sub_res
df["number_in_collection"] = df.sort_values("release_date" ).groupby("collection_name" ).cumcount() +1
df.loc[df["collection_name"]=="nashi", "number_in_collection"] = 0 
<feature_engineering>
%%time
do_full = False
n = 20
if Path('.. /input/rsna-str-pulmonary-embolism-detection/train' ).exists() and not do_full:
test_study_dirnames = [datapath/'test'/o for o in test_df['StudyInstanceUID'].unique() ]
test_study_dirnames = np.random.choice(test_study_dirnames, n, replace=False)
 sub_res = []
for study_dirname in test_study_dirnames:
sop_ids, seq_img_preds, seq_exam_preds = predict_study(study_dirname)
 study_res = get_study_res(study_dirname.stem, sop_ids, seq_img_preds, seq_exam_preds)
 sub_res += study_res
df["collection_av_logrevenue"] = [ df.loc[(df["collection_name"]==row["collection_name"])& 
(df["number_in_collection"]<row["number_in_collection"]),
"log_revenue"].mean()
for key,row in df.iterrows() ] 
<feature_engineering>
final_sub_df = pd.DataFrame(sub_res, columns=['id', 'label'])
 final_sub_df['label'] = np.clip(final_sub_df['label'], 0.0001, 0.9999 )
df.loc[df["collection_name"]=="nashi", "collection_av_logrevenue"] = df.loc[df["collection_name"]=="nashi", "log_revenue"].mean()<feature_engineering>
final_sub_df.to_csv("submission.csv", index=False )
collection_mean = df.loc[df["collection_name"]!="nashi", "log_revenue"].mean()
df["collection_av_logrevenue"] = df["collection_av_logrevenue"].fillna(collection_mean)
<concatenate>
def check_consistency(sub, test):

 df_tmp = sub.loc[sub.id.str.contains('_', regex = False)].reset_index(drop = True)
 df_tmp['StudyInstanceUID'] = df_tmp['id'].apply(lambda x:str(x ).split('_')[0])
 df_tmp['label_type'] = df_tmp['id'].apply(lambda x:'_'.join(str(x ).split('_')[1:]))
df_exam = df_tmp.pivot(index = 'StudyInstanceUID', columns = 'label_type', values = 'label')
 df_image = sub.loc[sub.id.isin(test.SOPInstanceUID)].reset_index(drop = True)
 df_image = df_image.merge(test, how = 'left', left_on = 'id', right_on = 'SOPInstanceUID')
 df_image.rename(columns = {"label": "pe_present_on_image"}, inplace = True)
 del df_image['id']
df = df_exam.merge(df_image, how = 'left', on = 'StudyInstanceUID')
 ids = ['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID']
labels = [c for c in df.columns if c not in ids]
df = df[ids + labels]
df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID'] ).pe_present_on_image.max())
 df_pos = df.loc[df.positive_images_in_exam > 0.5]
df_neg = df.loc[df.positive_images_in_exam <= 0.5]
rule1a = df_pos.loc[(( df_pos.rv_lv_ratio_lt_1 > 0.5)& 
(df_pos.rv_lv_ratio_gte_1 > 0.5)) | 
(( df_pos.rv_lv_ratio_lt_1 <= 0.5)& 
(df_pos.rv_lv_ratio_gte_1 <= 0.5)) ].reset_index(drop = True)
 rule1a['broken_rule'] = '1a'
rule1b = df_pos.loc[(df_pos.central_pe <= 0.5)& 
(df_pos.rightsided_pe <= 0.5)& 
(df_pos.leftsided_pe <= 0.5)].reset_index(drop = True)
 rule1b['broken_rule'] = '1b'
rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5)& 
(df_pos.chronic_pe > 0.5)].reset_index(drop = True)
 rule1c['broken_rule'] = '1c'
rule1d = df_pos.loc[(df_pos.indeterminate > 0.5)| 
(df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)
 rule1d['broken_rule'] = '1d'
rule2a = df_neg.loc[(( df_neg.indeterminate > 0.5)& 
(df_neg.negative_exam_for_pe > 0.5)) | 
(( df_neg.indeterminate <= 0.5)& 
(df_neg.negative_exam_for_pe <= 0.5)) ].reset_index(drop = True)
 rule2a['broken_rule'] = '2a'
rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1 > 0.5)| 
(df_neg.rv_lv_ratio_gte_1 > 0.5)| 
(df_neg.central_pe > 0.5)| 
(df_neg.rightsided_pe > 0.5)| 
(df_neg.leftsided_pe > 0.5)| 
(df_neg.acute_and_chronic_pe > 0.5)| 
(df_neg.chronic_pe > 0.5)].reset_index(drop = True)
 rule2b['broken_rule'] = '2b'
errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)
 print('Found', len(errors), 'inconsistent predictions')
 if len(errors)> 0:
print(errors.broken_rule.value_counts())
 return errors
df_features = pd.concat(dfdic_feature, axis=1 )<data_type_conversions>
%%time
consistency_df0 = check_consistency(final_sub_df, test_df )
df[["original_language", "collection_name"]] = df[["original_language", "collection_name"]].astype("category" )<categorify>
consistency_df0['broken_rule'].value_counts()
df_use = pd.get_dummies(df_use )<load_from_csv>
from scipy.special import softmax
train_add = pd.read_csv('.. /input/tmdb-competition-additional-features/TrainAdditionalFeatures.csv')
 test_add = pd.read_csv('.. /input/tmdb-competition-additional-features/TestAdditionalFeatures.csv')
 train_add.head()<merge>
def solve_rule_1b(sub, errors, log=False):
studies_break_rule = errors.query("(central_pe <= 0.5)&(rightsided_pe <= 0.5)&(leftsided_pe <= 0.5)")['StudyInstanceUID'].unique()
if log:
print(studies_break_rule,studies_break_rule.shape)
 for i in studies_break_rule:
idx = sub[sub['id'].str.contains(i)].index.values
old = sub.loc[idx[[3,5,7]], 'label'].values
best = np.argmax(old)
 new = old.copy()
if(old[0] == 0.5)and(old[1] == 0.5)and(old[2] == 0.5):
pass
else:
new[best] = new[best] + 0.5
new = np.clip(new , 0, 0.51)
 if log:
print(old, new)
 sub.loc[idx[[3,5,7]], 'label'] = new
df = pd.merge(df, pd.concat([train_add, test_add]), on="imdb_id", how="left" )<feature_engineering>
def solve_rule_1c(sub, errors, log=False):
studies_break_rule = errors.query("(acute_and_chronic_pe > 0.5)&(chronic_pe > 0.5)")['StudyInstanceUID'].unique()
if log:
print(studies_break_rule, studies_break_rule.shape)
 for i in studies_break_rule:
idx = sub[sub['id'].str.contains(i)].index.values
old = sub.loc[idx[[4,6]], 'label'].values
new = softmax(old)
 if log:
print(old , new)
 sub.loc[idx[[4,6]], 'label'] == new
add_cols = ["popularity2", "rating", "totalVotes"]
df[add_cols] = df[add_cols].fillna(df[add_cols].mean() )<load_from_csv>
def solve_negative(sub, errors):
negative_sopids = errors.query("negative_exam_for_pe > 0.5")['SOPInstanceUID'].values
neg_clip_idxs = sub[sub['id'].isin(negative_sopids)].index
sub.loc[neg_clip_idxs, 'label'] = np.clip(sub.loc[neg_clip_idxs, 'label'], 0, 0.5 )
train2 = pd.read_csv('.. /input/tmdb-box-office-prediction-more-training-data/additionalTrainData.csv')
 train3 = pd.read_csv('.. /input/tmdb-box-office-prediction-more-training-data/trainV3.csv')
 train3.head()<categorify>
%%time
solve_negative(sub=final_sub_df, errors=consistency_df0 )
def lower_text(text):
return text.lower()
def remove_punct(text):
text = text.replace('-', ' ')
 table=str.maketrans('','',string.punctuation)
 return text.translate(table)
 def remove_stopwords(words, stopwords):
words = [word for word in words if word not in stopwords]
return words<feature_engineering>
%%time
consistency_df0 = check_consistency(sub=final_sub_df, test=test_df )
<define_variables>
%%time
solve_rule_1a(sub=final_sub_df, errors=consistency_df0, log=False)
 solve_rule_1b(sub=final_sub_df, errors=consistency_df0, log=False)
 solve_rule_1c(sub=final_sub_df, errors=consistency_df0, log=False )
no_english_overview_id = [157, 2863, 4616]
no_english_tagline_id = [3255, 3777, 4937]<feature_engineering>
consistency_df_final = check_consistency(final_sub_df, test_df )
df['overview_word_count'] = df['overview'].apply(lambda x: len(str(x ).split())) 
<feature_engineering>
submit_fixed = True
if submit_fixed:
final_sub_df.to_csv("submission.csv", index=False )
<feature_engineering><EOS> 
final_sub_df['label_prefix'] = final_sub_df.apply(lambda o: "_".join(o['id'].split("_")[1:]), 1)
 stats = final_sub_df.groupby("label_prefix" ).agg(['min', 'max', 'mean', 'median'])
 stats