Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
%matplotlib inline
sns.set(palette=sns.color_palette('Set2',9))
input_pad_idx = None<choose_model_class>
class AWD_LSTM(Module):
"AWD-LSTM inspired by https://arxiv.org/abs/1708.02182"
initrange=0.1
def __init__(self, emb_sz,n_hid, n_layers, hidden_p=0.2, input_p=0.6, weight_p=0.5, bidir=False):
store_attr('emb_sz,n_hid,n_layers')
 self.bs = 1
self.n_dir = 2 if bidir else 1
self.rnns = nn.ModuleList([self._one_rnn(emb_sz if l == 0 else n_hid,(n_hid)//self.n_dir, bidir, weight_p, l)for l in range(n_layers)])
 self.input_dp = RNNDropout(input_p)
 self.hidden_dps = nn.ModuleList([RNNDropout(hidden_p)for l in range(n_layers)])
 self.reset()
def forward(self, x, from_embeds=False):
if from_embeds: inp = x
else: inp = combined_embeddings[x].to(device)
 bs,sl = inp.shape[:2]
if bs!=self.bs: self._change_hidden(bs)
 output = self.input_dp(inp)
 new_hidden = []
for l,(rnn,hid_dp)in enumerate(zip(self.rnns, self.hidden_dps)) :
output, new_h = rnn(output, self.hidden[l])
 new_hidden.append(new_h)
 if l != self.n_layers - 1: output = hid_dp(output)
 self.hidden = to_detach(new_hidden, cpu=False, gather=False)
 return output
def _change_hidden(self, bs):
self.hidden = [self._change_one_hidden(l, bs)for l in range(self.n_layers)]
self.bs = bs
def _one_rnn(self, n_in, n_out, bidir, weight_p, l):
"Return one of the inner rnn"
rnn = nn.LSTM(n_in, n_out, 1, batch_first=True, bidirectional=bidir, bias=False)
 return WeightDropout(rnn, weight_p)
 def _one_hidden(self, l):
"Return one hidden state"
nh =(self.n_hid)// self.n_dir
return(one_param(self ).new_zeros(self.n_dir, self.bs, nh ).to(device), one_param(self ).new_zeros(self.n_dir, self.bs, nh ).to(device))
def _change_one_hidden(self, l, bs):
if self.bs < bs:
nh =(self.n_hid)// self.n_dir
return tuple(torch.cat([h, h.new_zeros(self.n_dir, bs-self.bs, nh)], dim=1)for h in self.hidden[l])
 if self.bs > bs: return(self.hidden[l][0][:,:bs].contiguous() , self.hidden[l][1][:,:bs].contiguous())
 return self.hidden[l]
def reset(self):
"Reset the hidden states"
[r.reset() for r in self.rnns if hasattr(r, 'reset')]
self.hidden = [self._one_hidden(l)for l in range(self.n_layers)]<categorify>
print(sns.__version__ )
lstm_width = 512
layers = [lstm_width * 3] + [lstm_width] + [9]
class MultiHeadedSequenceClassifier(Module):
"dim: input sequence feature dim"
def __init__(self, bptt=72, input_pad_idx=input_pad_idx, n_meta=1, dim=1024):
store_attr('input_pad_idx')
 self.awd_lstm = AWD_LSTM(dim+n_meta, lstm_width, 2, bidir=True)
 self.encoder = SentenceEncoder(bptt=bptt, module=self.awd_lstm, pad_idx=input_pad_idx)
 self.seq_head = LinearDecoder(1, lstm_width, bias=True)
 self.exam_head = PoolingLinearClassifier(layers, ps=[0.4, 0.1], bptt=bptt)
 def forward(self, x):
out, mask = self.encoder(x)
 seq_cls_out,_,_ = self.seq_head(out)
 seq_cls_out = seq_cls_out.squeeze(-1)
 exam_out,_,_ = self.exam_head(( out,mask))
return(seq_cls_out, exam_out)
 def predict(self, x):
out = self.awd_lstm(x, from_embeds=True)
 seq_cls_out,_,_ = self.seq_head(out)
 seq_cls_out = seq_cls_out.squeeze(-1)
 mask = torch.zeros(x.shape[:-1] ).bool().to(device)
 exam_out,_,_ = self.exam_head(( out, mask))
return(seq_cls_out, exam_out)
<categorify>
train_path='/kaggle/input/titanic/train.csv'
test_path='/kaggle/input/titanic/test.csv'
titanic_train=pd.read_csv(train_path)
 titanic_test=pd.read_csv(test_path )
layers1 = [512 * 3] + [512] + [12]
class MultiHeadedSoftmaxSequenceClassifier(Module):
"dim: input sequence feature dim"
def __init__(self, bptt=72, input_pad_idx=input_pad_idx, n_meta=2, dim=1024):
store_attr('input_pad_idx')
 self.awd_lstm = AWD_LSTM(dim+n_meta, 512, 2, bidir=True)
 self.encoder = SentenceEncoder(bptt=bptt, module=self.awd_lstm, pad_idx=input_pad_idx)
 self.seq_head = LinearDecoder(1, 512, bias=True)
 self.exam_head = PoolingLinearClassifier(layers1, ps=[0.4, 0.1], bptt=bptt)
 def forward(self, x):
out, mask = self.encoder(x)
 seq_cls_out,_,_ = self.seq_head(out)
 seq_cls_out = seq_cls_out.squeeze(-1)
 exam_out,_,_ = self.exam_head(( out,mask))
posneg_out, rvlv_out, lrc_out, chroacute_out =(exam_out[:,:3],
exam_out[:,3:6],
exam_out[:,6:9],
exam_out[:,9:])
 return(seq_cls_out, posneg_out, rvlv_out, lrc_out, chroacute_out )<load_pretrained>
titanic_train['Cabin_t'] = titanic_train['Cabin_t'].replace(['A', 'B', 'C','T'], 'ABCT')
 titanic_train['Cabin_t'] = titanic_train['Cabin_t'].replace(['D', 'E'], 'DE')
 titanic_train['Cabin_t'] = titanic_train['Cabin_t'].replace(['F', 'G'], 'FG' )
seq_model0 = SequentialRNN(MultiHeadedSequenceClassifier(bptt=256, dim=1024, n_meta=2))
seq_model0.load_state_dict(torch.load(seqmodelpath/"nometa_sequence_with_preds_fulldata_fold0.pth")) ;
seq_model2 = SequentialRNN(MultiHeadedSequenceClassifier(bptt=256, dim=1024, n_meta=2))
seq_model2.load_state_dict(torch.load(seqmodelpath/"nometa_sequence_with_preds_fulldata_fold2.pth")) ;
seq_model3 = SequentialRNN(MultiHeadedSequenceClassifier(bptt=256, dim=1024, n_meta=2))
seq_model3.load_state_dict(torch.load(seqmodelpath/"nometa_sequence_with_preds_fulldata_fold3.pth")) ;
seq_model4 = SequentialRNN(MultiHeadedSequenceClassifier(bptt=256, dim=1024, n_meta=2))
seq_model4.load_state_dict(torch.load(seqmodelpath/"nometa_sequence_with_preds_fulldata_fold4.pth")) ;<predict_on_test>
titanic_train['Name_m']=titanic_train['Name'].str.split(pat=', ',n=1,expand=True)[1].str.split(pat='.',n=1,expand=True)[0]
def Name_transform(x):
if x=='Mr':
return 'Mr'
elif x=='Mrs':
return 'Mrs'
elif x=='Miss':
return 'Miss'
else:
return 'etc'
titanic_train['Name_M']=titanic_train['Name_m'].apply(Name_transform )
seq_model0 = seq_model0[0].to(device ).eval()
seq_model2 = seq_model2[0].to(device ).eval()
seq_model3 = seq_model3[0].to(device ).eval()
seq_model4 = seq_model4[0].to(device ).eval()<define_variables>
titanic_train['Ticket_Freq']=titanic_train.groupby('Ticket')['Ticket'].transform('count' )
mean_std_dict = {
'scaled_position': [0.5078721739409284, 0.29139548181397823]
}<import_modules>
titanic_train=pd.read_csv(train_path)
 titanic_test=pd.read_csv(test_path)
 target=titanic_train['Survived']
Id=titanic_test[['PassengerId']]
from fastai.medical.imaging import *
import pydicom<categorify>
class NullTransformer(BaseEstimator,TransformerMixin):
def fit(self,df,y=None):
return self
def transform(self,df):
df['Embarked'].fillna('S',inplace=True)
 df['Cabin'].fillna('X',inplace=True)
 missing_Fare_index=list(df['Fare'][df['Fare'].isnull() ].index)
 for index in missing_Fare_index:
if df['Pclass'][index]==1:
df['Fare'].iloc[index]=df.groupby('Pclass')['Fare'].median() [1]
elif df['Pclass'][index]==2:
df['Fare'].iloc[index]=df.groupby('Pclass')['Fare'].median() [2]
elif df['Pclass'][index]==3:
df['Fare'].iloc[index]=df.groupby('Pclass')['Fare'].median() [3]
index_NaN_age = list(df["Age"][df["Age"].isnull() ].index)
 for index in index_NaN_age :
if df['Pclass'][index]==1:
df['Age'].iloc[index]=df.groupby('Pclass')['Age'].median() [1]
elif df['Pclass'][index]==2:
df['Age'].iloc[index]=df.groupby('Pclass')['Age'].median() [2]
elif df['Pclass'][index]==3:
df['Age'].iloc[index]=df.groupby('Pclass')['Age'].median() [3]
return df
class EmbeddingHook:
def __init__(self, m):
self.embeddings, self.m = tensor([] ).to(device), m
if len(m._forward_hooks)> 0: self.reset()
self.hook = Hook(m, self.hook_fn, cpu=False)
 def hook_fn(self, m, inp, out):
"Stack and save computed embeddings"
self.embeddings = torch.cat([self.embeddings, out])
 def reset(self):
self.m._forward_hooks = OrderedDict()<define_variables>
class FeatureExtraction(BaseEstimator,TransformerMixin):
def fit(self,df,y=None):
return self
def transform(self,df):
df['Family']=df['SibSp']+df['Parch']+1
def groupfamily(x):
if x==1:
return 1
elif(2<=x)&(x<=4):
return 2
elif(5<=x)&(x<=6):
return 3
else:
return 4
df['Family']=df['Family'].apply(groupfamily)
 df.drop(['SibSp','Parch','PassengerId'],inplace=True,axis=1)
 if 'Survived' in list(df.keys()):
df.drop(['Survived'],inplace=True,axis=1)
 df['Ticket_freq']=df.groupby('Ticket')['Ticket'].transform('count')
 df.drop(['Ticket'],inplace=True,axis=1)
 df['Cabin']=df['Cabin'].str.get(i=0)
 df['Cabin'] = df['Cabin'].replace(['A', 'B', 'C','T'], 'ABCT')
 df['Cabin'] = df['Cabin'].replace(['D', 'E'], 'DE')
 df['Cabin'] = df['Cabin'].replace(['F', 'G'], 'FG')
 df['Name']=df['Name'].str.split(pat=', ',n=1,expand=True)[1].str.split(pat='.',n=1,expand=True)[0]
def Name_transform(x):
if x=='Mr':
return 'Mr'
elif x=='Mrs':
return 'Mrs'
elif x=='Miss':
return 'Miss'
else:
return 'etc'
df['Name']=df['Name'].apply(Name_transform)
 AGE_BINS=[-1,5,17,20,24,28,32,40,48,100]
df['Age_cat']=pd.cut(df['Age'],AGE_BINS,labels=[0.,1.,2.,3.,4.,5.,6.,7.,8.])
 FARE_BINS=[-1,7.35, 7.82, 8, 10, 13, 23,30, 45, 80, 150, 1000]
df['Fare_cat']=pd.cut(df['Fare'],bins=FARE_BINS,labels=[0.,1.,2.,3.,4.,5.,6.,7.,8.,9.,10.])
 df.drop(['Age','Fare'],inplace=True,axis=1)
 return df
meta_feat_cols = ['scaled_position']<normalization>
attribs=['Pclass','Name','Sex','Age','Ticket','Fare','Cabin','Embarked','SibSp','Parch']
num_attribs=['Pclass','Age_cat','Fare_cat',"Ticket_freq"]
cat_attribs=['Name','Sex','Cabin','Embarked','Family']
pipeline1=Pipeline([ 
('NT',NullTransformer()), 
('FE',FeatureExtraction())
 ])
 train=pipeline1.fit_transform(titanic_train)
 test=pipeline1.transform(titanic_test)
 train=pd.get_dummies(train,columns=cat_attribs)
 test=pd.get_dummies(test,columns=cat_attribs)
def minmax_scaler(o): return(o - min(o)) /(max(o)- min(o))<load_pretrained>
train=train.to_numpy()
test=test.to_numpy()
def read_dcm_img_v2(dcm, windows=windows):
"Read single slice in RGB"
return [dcm.windowed(*w)for w in windows]<predict_on_test>
kfolds=StratifiedKFold(n_splits=3)
 scores=[]
svc=SVC(random_state=42)
 rdf=RandomForestClassifier(random_state=42)
 ada=AdaBoostClassifier(random_state=42)
 log=LogisticRegression(random_state=42)
 grb=GradientBoostingClassifier(random_state=42)
 dct=DecisionTreeClassifier(random_state=42)
 ext=ExtraTreesClassifier(random_state=42)
 xgb=XGBRFClassifier(random_state=42)
 xgbc=XGBClassifier(random_state=42)
 models=[svc,rdf,ada,log,grb,dct,ext,xgb,xgbc]
for model in models:
scores.append(cross_val_score(model,train,target,cv=kfolds,n_jobs=-1))
scores_mean=np.mean(scores,axis=1 )
def predict_study(study_dirname):
study_df = test_df.query(f'StudyInstanceUID == "{study_dirname.stem}"')
 sop_ids = study_df['SOPInstanceUID'].values
study_files = str(testdatapath)+ "/" + study_df['StudyInstanceUID'] + "/" + study_df['SeriesInstanceUID'] + '/' + sop_ids + '.dcm'
dcm_ds = array([pydicom.read_file(o)for o in study_files])
 z_positions = array([int(o.ImagePositionPatient[-1])for o in dcm_ds])
 sortidxs = np.argsort(z_positions)
 sop_ids = sop_ids[sortidxs]
dcm_ds = dcm_ds[sortidxs]
z_positions = z_positions[sortidxs]
imgs = []
for o in dcm_ds: imgs += read_dcm_img_v2(o)
 h,w = imgs[0].size()
imgs = torch.stack(imgs ).view(-1,3,h,w)
 meta_embeddings = tensor(( minmax_scaler(z_positions)- mean)/std ).unsqueeze(1)
 emb_hook0 = EmbeddingHook(model0[1][1])
 emb_hook2 = EmbeddingHook(model2[1][1])
 emb_hook3 = EmbeddingHook(model3[1][1])
 emb_hook4 = EmbeddingHook(model4[1][1])
 with torch.no_grad() :
test_dl = learn2d_fold0.dls.test_dl(imgs.numpy() , bs=32, num_workers=2)
 outs0, outs1, outs2, outs3, outs4, outs5 = [],[],[],[],[],[]
for xb in test_dl:
out0 = model0(*xb); outs0.append(out0)
 out2 = model2(*xb); outs2.append(out2)
 out3 = model3(*xb); outs3.append(out3)
 out4 = model4(*xb); outs4.append(out4)
 outs0 = torch.cat(outs0)[:,1].view(-1,1)
 outs2 = torch.cat(outs2)[:,1].view(-1,1)
 outs3 = torch.cat(outs3)[:,1].view(-1,1)
 outs4 = torch.cat(outs4)[:,1].view(-1,1)
 seq_inp0 = torch.cat([emb_hook0.embeddings, meta_embeddings.to(device), outs0],1)
 seq_inp2 = torch.cat([emb_hook2.embeddings, meta_embeddings.to(device), outs2],1)
 seq_inp3 = torch.cat([emb_hook3.embeddings, meta_embeddings.to(device), outs3],1)
 seq_inp4 = torch.cat([emb_hook4.embeddings, meta_embeddings.to(device), outs4],1)
 seq_img_preds0, seq_exam_preds0 = seq_model0.predict(seq_inp0[None,...])
 seq_img_preds2, seq_exam_preds2 = seq_model2.predict(seq_inp2[None,...])
 seq_img_preds3, seq_exam_preds3 = seq_model3.predict(seq_inp3[None,...])
 seq_img_preds4, seq_exam_preds4 = seq_model4.predict(seq_inp4[None,...])
 seq_img_preds0, seq_exam_preds0 = seq_img_preds0.sigmoid() , seq_exam_preds0.sigmoid()
seq_img_preds2, seq_exam_preds2 = seq_img_preds2.sigmoid() , seq_exam_preds2.sigmoid()
seq_img_preds3, seq_exam_preds3 = seq_img_preds3.sigmoid() , seq_exam_preds3.sigmoid()
seq_img_preds4, seq_exam_preds4 = seq_img_preds4.sigmoid() , seq_exam_preds4.sigmoid()
fold_wgts = array([1,0,1,1,1])
 seq_img_preds = to_cpu(( seq_img_preds0 *(fold_wgts[0])+
seq_img_preds2 *(fold_wgts[2])+
seq_img_preds3 *(fold_wgts[3])+
seq_img_preds4 *(fold_wgts[4])
)/ sum(fold_wgts)) [0]
seq_exam_preds = to_cpu(( seq_exam_preds0 *(fold_wgts[0])+
seq_exam_preds2 *(fold_wgts[2])+
seq_exam_preds3 *(fold_wgts[3])+
seq_exam_preds4 *(fold_wgts[4])
)/ sum(fold_wgts)) [0]
return(sop_ids, seq_img_preds, seq_exam_preds )<compute_train_metric>
def grid_search(estimator,param,X,y):
grid=GridSearchCV(estimator, param, n_jobs=-1, cv=kfolds, return_train_score=True)
 grid.fit(X,y)
 grid_results=grid.cv_results_
print('
BestParams and Score :
',grid.best_params_,'
',grid.best_score_)
 return grid.best_estimator_ , grid, estimator
def get_study_res(sid, sop_ids, img_preds, exam_preds):
"Get preds from 2D and 3D cnn models"
sub_res = []
for sopid, p in zip(sop_ids, to_np(img_preds)) :
sub_res.append(( sopid, p))
target_cols = [
'negative_exam_for_pe',
'rv_lv_ratio_gte_1',
'rv_lv_ratio_lt_1',
'leftsided_pe',
'chronic_pe',
'rightsided_pe',
'acute_and_chronic_pe',
'central_pe',
'indeterminate'
]
for tcol, p in zip(target_cols, to_np(exam_preds)) :
sub_res.append(( f"{sid}_{tcol}", p))
return sub_res<predict_on_test>
def random_grid_search(estimator,param_d,X,y,n):
grid=RandomizedSearchCV(estimator, param_d, n_jobs=-1, cv=kfolds, return_train_score=True, n_iter=n)
 grid.fit(X,y)
 grid_results=grid.cv_results_
print('
BestParams and Score :
',grid.best_params_,'
',grid.best_score_)
 return grid.best_estimator_ , grid, estimator
%%time
do_full = False
n = 20
if Path('.. /input/rsna-str-pulmonary-embolism-detection/train' ).exists() and not do_full:
test_study_dirnames = [datapath/'test'/o for o in test_df['StudyInstanceUID'].unique() ]
test_study_dirnames = np.random.choice(test_study_dirnames, n, replace=False)
 sub_res = []
for study_dirname in test_study_dirnames:
sop_ids, seq_img_preds, seq_exam_preds = predict_study(study_dirname)
 study_res = get_study_res(study_dirname.stem, sop_ids, seq_img_preds, seq_exam_preds)
 sub_res += study_res<prepare_output>
param_grid = [
{'C':[0.6, 0.8,1,1.15,1.2,1.23,1.4], 'kernel':['rbf'], 'gamma':[0.1] }
]
svc_best, svc_grid, svc=grid_search(SVC(random_state=42, probability=True),param_grid,X=train,y=target )
final_sub_df = pd.DataFrame(sub_res, columns=['id', 'label'])
 final_sub_df['label'] = np.clip(final_sub_df['label'], 0.0001, 0.9999 )<save_to_csv>
param_grid = [
{'C': uniform(0.9,0.3), 'kernel':['rbf'], 'gamma':[0.1] }
]
svc_best, svc_grid, svc =random_grid_search(SVC(random_state=42, probability=True),param_grid,X=train,y=target,n=20 )
final_sub_df.to_csv("submission.csv", index=False )<feature_engineering>
param_grid = [
{'learning_rate':[0.1], 'n_estimators':[30,50,100,200,240], 'max_depth':[2,3,4] }
]
gbc_best, gbc_grid, gbc =grid_search(GradientBoostingClassifier(random_state=42),param_grid,X=train,y=target )
def check_consistency(sub, test):

 df_tmp = sub.loc[sub.id.str.contains('_', regex = False)].reset_index(drop = True)
 df_tmp['StudyInstanceUID'] = df_tmp['id'].apply(lambda x:str(x ).split('_')[0])
 df_tmp['label_type'] = df_tmp['id'].apply(lambda x:'_'.join(str(x ).split('_')[1:]))
df_exam = df_tmp.pivot(index = 'StudyInstanceUID', columns = 'label_type', values = 'label')
 df_image = sub.loc[sub.id.isin(test.SOPInstanceUID)].reset_index(drop = True)
 df_image = df_image.merge(test, how = 'left', left_on = 'id', right_on = 'SOPInstanceUID')
 df_image.rename(columns = {"label": "pe_present_on_image"}, inplace = True)
 del df_image['id']
df = df_exam.merge(df_image, how = 'left', on = 'StudyInstanceUID')
 ids = ['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID']
labels = [c for c in df.columns if c not in ids]
df = df[ids + labels]
df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID'] ).pe_present_on_image.max())
 df_pos = df.loc[df.positive_images_in_exam > 0.5]
df_neg = df.loc[df.positive_images_in_exam <= 0.5]
rule1a = df_pos.loc[(( df_pos.rv_lv_ratio_lt_1 > 0.5)& 
(df_pos.rv_lv_ratio_gte_1 > 0.5)) | 
(( df_pos.rv_lv_ratio_lt_1 <= 0.5)& 
(df_pos.rv_lv_ratio_gte_1 <= 0.5)) ].reset_index(drop = True)
 rule1a['broken_rule'] = '1a'
rule1b = df_pos.loc[(df_pos.central_pe <= 0.5)& 
(df_pos.rightsided_pe <= 0.5)& 
(df_pos.leftsided_pe <= 0.5)].reset_index(drop = True)
 rule1b['broken_rule'] = '1b'
rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5)& 
(df_pos.chronic_pe > 0.5)].reset_index(drop = True)
 rule1c['broken_rule'] = '1c'
rule1d = df_pos.loc[(df_pos.indeterminate > 0.5)| 
(df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)
 rule1d['broken_rule'] = '1d'
rule2a = df_neg.loc[(( df_neg.indeterminate > 0.5)& 
(df_neg.negative_exam_for_pe > 0.5)) | 
(( df_neg.indeterminate <= 0.5)& 
(df_neg.negative_exam_for_pe <= 0.5)) ].reset_index(drop = True)
 rule2a['broken_rule'] = '2a'
rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1 > 0.5)| 
(df_neg.rv_lv_ratio_gte_1 > 0.5)| 
(df_neg.central_pe > 0.5)| 
(df_neg.rightsided_pe > 0.5)| 
(df_neg.leftsided_pe > 0.5)| 
(df_neg.acute_and_chronic_pe > 0.5)| 
(df_neg.chronic_pe > 0.5)].reset_index(drop = True)
 rule2b['broken_rule'] = '2b'
errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)
 print('Found', len(errors), 'inconsistent predictions')
 if len(errors)> 0:
print(errors.broken_rule.value_counts())
 return errors<compute_test_metric>
param_grid = [
{'learning_rate':[0.1], 'n_estimators': randint(80,110), 'max_depth':[3] }
]
gbc_best, gbc_grid, gbc =random_grid_search(GradientBoostingClassifier(random_state=42),param_grid,X=train,y=target,n=20 )
%%time
consistency_df0 = check_consistency(final_sub_df, test_df )<count_values>
param_grid = [
{'penalty':['l1','l2'],'C':[0.1,0.8,0.9,1,1.1,1.2,1.3,5], 'tol':[1e-4,1e-3],'solver':['liblinear']}
]
log_best, log_grid, log =grid_search(LogisticRegression(random_state=42),param_grid,X=train,y=target )
consistency_df0['broken_rule'].value_counts()<import_modules>
param_grid = [
{'penalty':['l1'],'C': uniform(0.7,0.3), 'tol':[1e-4],'solver':['liblinear']}
]
log_best, log_grid, log =random_grid_search(LogisticRegression(random_state=42),param_grid,X=train,y=target,n=20 )
from scipy.special import softmax<find_best_params>
param_grid = [
{'n_estimators':randint(20,60),'max_depth':[4], 'gamma':[0.1]}
]
xgb_best, xgb_grid, xgb =random_grid_search(XGBRFClassifier(random_state=42),param_grid,X=train,y=target,n=20 )
def solve_rule_1b(sub, errors, log=False):
studies_break_rule = errors.query("(central_pe <= 0.5)&(rightsided_pe <= 0.5)&(leftsided_pe <= 0.5)")['StudyInstanceUID'].unique()
if log:
print(studies_break_rule,studies_break_rule.shape)
 for i in studies_break_rule:
idx = sub[sub['id'].str.contains(i)].index.values
old = sub.loc[idx[[3,5,7]], 'label'].values
best = np.argmax(old)
 new = old.copy()
if(old[0] == 0.5)and(old[1] == 0.5)and(old[2] == 0.5):
pass
else:
new[best] = new[best] + 0.5
new = np.clip(new , 0, 0.51)
 if log:
print(old, new)
 sub.loc[idx[[3,5,7]], 'label'] = new<find_best_params>
param_grid = [
{'n_estimators': randint(70,120),'max_depth':[3], 'gamma':[0.1]}
]
xgbc_best, xgbc_grid, xgbc =random_grid_search(XGBClassifier(random_state=42),param_grid,X=train,y=target,n=20 )
def solve_rule_1c(sub, errors, log=False):
studies_break_rule = errors.query("(acute_and_chronic_pe > 0.5)&(chronic_pe > 0.5)")['StudyInstanceUID'].unique()
if log:
print(studies_break_rule, studies_break_rule.shape)
 for i in studies_break_rule:
idx = sub[sub['id'].str.contains(i)].index.values
old = sub.loc[idx[[4,6]], 'label'].values
new = softmax(old)
 if log:
print(old , new)
 sub.loc[idx[[4,6]], 'label'] == new<feature_engineering>
param_grid = [
{'n_estimators': [30,50,100,200,300],'learning_rate':[0.9,1.0,1.1,1.2,1.3]}
]
ada_best, ada_grid, ada =grid_search(AdaBoostClassifier(random_state=42),param_grid,X=train,y=target )
def solve_negative(sub, errors):
negative_sopids = errors.query("negative_exam_for_pe > 0.5")['SOPInstanceUID'].values
neg_clip_idxs = sub[sub['id'].isin(negative_sopids)].index
sub.loc[neg_clip_idxs, 'label'] = np.clip(sub.loc[neg_clip_idxs, 'label'], 0, 0.5 )<compute_train_metric>
param_grid = [
{'n_estimators': randint(20,50),'learning_rate':[1.3]}
]
ada_best, ada_grid, ada =random_grid_search(AdaBoostClassifier(random_state=42),param_grid,X=train,y=target,n=20 )
%%time
solve_negative(sub=final_sub_df, errors=consistency_df0 )<concatenate>
param_grid = [
{'n_estimators': [30,50,100,200,300],'max_depth':[2,3,4,5], 'criterion':['gini','entropy']}
]
rdf_best, rdf_grid, rdf =grid_search(RandomForestClassifier(random_state=42),param_grid,X=train,y=target )
%%time
consistency_df0 = check_consistency(sub=final_sub_df, test=test_df )<find_best_params>
param_grid = [
{'n_estimators': randint(200,400),'max_depth':[5], 'criterion':['gini']}
]
rdf_best, rdf_grid, rdf =random_grid_search(RandomForestClassifier(random_state=42),param_grid,X=train,y=target,n=20 )
%%time
solve_rule_1a(sub=final_sub_df, errors=consistency_df0, log=False)
 solve_rule_1b(sub=final_sub_df, errors=consistency_df0, log=False)
 solve_rule_1c(sub=final_sub_df, errors=consistency_df0, log=False )<compute_test_metric>
param_grid = [
{'n_estimators': randint(80,150),'max_depth':[5], 'criterion':['gini']}
]
ext_best, ext_grid, ext =random_grid_search(ExtraTreesClassifier(random_state=42),param_grid,X=train,y=target,n=20 )
consistency_df_final = check_consistency(final_sub_df, test_df )<save_to_csv>
models=[ 
('svc', svc_best), 
('log', log_best), 
('xgb',xgb_best), 
('gbc', gbc_best), 
('xgbc', xgbc_best), 
('ada', xgbc_best), 
('rdf',rdf_best), 
('ext',ext_best)
 ]
vot_hard = VotingClassifier(estimators=models, voting='hard', n_jobs=-1)
 vot_hard.fit(train,target)
 vot_soft = VotingClassifier(estimators=models, voting='soft', n_jobs=-1)
 vot_soft.fit(train,target)
 stack=StackingClassifier(estimators=models,cv=kfolds,n_jobs=-1,stack_method='predict_proba')
 stack.fit(train,target)
 prediction_hard=vot_hard.predict(test)
 prediction_soft=vot_soft.predict(test)
 prediction_stack=stack.predict(test)
submit_fixed = True
if submit_fixed:
final_sub_df.to_csv("submission.csv", index=False )<groupby>
kfolds=StratifiedKFold(n_splits=3)
 scores=[]
models=[vot_hard,vot_soft,stack]
for model in models:
scores.append(cross_val_score(model,train,target,cv=kfolds))
scores_mean=np.mean(scores,axis=1 )
final_sub_df['label_prefix'] = final_sub_df.apply(lambda o: "_".join(o['id'].split("_")[1:]), 1)
 stats = final_sub_df.groupby("label_prefix" ).agg(['min', 'max', 'mean', 'median'])
 stats<install_modules>
Survived_hard = pd.Series(prediction_hard, name="Survived")
 result_hard = pd.concat([Id,Survived_hard],axis=1)
 result_hard.to_csv("submission_hard.csv",index=False)
 Survived_soft = pd.Series(prediction_soft, name="Survived")
 result_soft = pd.concat([Id,Survived_soft],axis=1)
 result_soft.to_csv("submission_soft.csv",index=False)
 Survived_stack = pd.Series(prediction_stack, name="Survived")
 result_stack = pd.concat([Id,Survived_stack],axis=1)
 result_stack.to_csv("submission_stack.csv",index=False )