Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
%matplotlib inline
%%time
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_end_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=n_split,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting all Train for Outlier analysis...')
 preds_train = padded_model.predict([input_ids,attention_mask,token_type_ids],verbose=DISPLAY)
 preds_start_train += preds_train[0]/skf.n_splits
preds_end_train += preds_train[1]/skf.n_splits
print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<compute_test_metric>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
print(jac )<string_transform>
train = train.drop(['PassengerId','Name','SibSp','Parch'], axis = 1)
 test = test.drop(['Name','SibSp','Parch'], axis = 1 )
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st )<save_to_csv>
print("TRAIN DATA:")
 train.isnull().sum()
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 test.sample(10 )<define_variables>
print("TEST DATA:")
 test.isnull().sum()
all = []
start = []
end = []
start_pred = []
end_pred = []
for k in range(input_ids.shape[0]):
a = np.argmax(preds_start_train[k,])
 b = np.argmax(preds_end_train[k,])
 start.append(np.argmax(start_tokens[k]))
end.append(np.argmax(end_tokens[k]))
if a>b:
st = train.loc[k,'text']
start_pred.append(0)
 end_pred.append(len(st))
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 start_pred.append(a)
 end_pred.append(b)
 all.append(st)
 train['start'] = start
train['end'] = end
train['start_pred'] = start_pred
train['end_pred'] = end_pred
train['selected_text_pred'] = all
train.sample(10 )<compute_train_metric>
numerical = ['Pclass','Age','Is_alone','Fare']
categorical = ['Sex','Ticket','Cabin', 'Embarked']
def metric_tse(df,col1,col2):
return df.apply(lambda x: jaccard(x[col1],x[col2]),axis=1 )<feature_engineering>
features = numerical + categorical
target = ['Survived']
print('Features:', features, '
Target:', target )
train = train.replace({'sentiment': {'negative': -1, 'neutral': 0, 'positive': 1}})
 train['len_text'] = train['text'].str.len()
train['len_selected_text'] = train['selected_text'].str.len()
train['diff_num'] = train['end']-train['start']
train['share'] = train['len_selected_text']/train['len_text']<feature_engineering>
numerical_transformer = Pipeline(
 steps=[('iterative', IterativeImputer(max_iter = 10, random_state=0)) , 
('scaler', StandardScaler())])
 categorical_transformer = Pipeline(
 steps=[('imputer', SimpleImputer(strategy='most_frequent')) , 
('onehot', OneHotEncoder(handle_unknown='ignore')) ])
 preprocessor = ColumnTransformer(
 transformers=[('num', numerical_transformer, numerical), 
('cat', categorical_transformer, categorical)] )
train['selected_text_pred'] = train['selected_text_pred'].map(lambda x: x.lstrip(' '))
train['len_selected_text_pred'] = train['selected_text_pred'].str.len()
train['diff_num_pred'] = train['end_pred']-train['start_pred']
train['share_pred'] = train['len_selected_text_pred']/train['len_text']
train['len_equal'] = 0
train.loc[(train['start'] == train['start_pred'])&(train['end'] == train['end_pred']), 'len_equal'] = 1
train['metric'] = metric_tse(train,'selected_text','selected_text_pred')
 train['res'] = 0
train.loc[train['metric'] == 1, 'res'] = 1<string_transform>
observations = pd.DataFrame()
classifiers = ['Linear SVM', 'Radial SVM', 'LogisticRegression',
'RandomForestClassifier', 'AdaBoostClassifier',
'XGBoostClassifier', 'KNeighborsClassifier']
models = [svm.SVC(kernel='linear'),
svm.SVC(kernel='rbf'),
LogisticRegression() ,
RandomForestClassifier(n_estimators=200, random_state=0),
AdaBoostClassifier(random_state = 0),
xgb.XGBClassifier(n_estimators=100),
KNeighborsClassifier()
]
j = 0
for i in models:
model = i
cv = KFold(n_splits=5, random_state=0, shuffle=True)
 pipe = Pipeline(steps=[('preprocessor', preprocessor), 
('model', model)])
 observations[classifiers[j]] =(cross_val_score(pipe, train[features], np.ravel(train[target]), scoring='accuracy', cv=cv))
j = j+1
def rep_3chr(text):
chr3 = 0
for word in text.split() :
for c in set(word):
if word.rfind(c+c+c)> -1:
chr3 = 1
return chr3<feature_engineering>
mean = pd.DataFrame(observations.mean() , index= classifiers)
 observations = pd.concat([observations,mean.T])
 observations.index=['Fold 1','Fold 2','Fold 3','Fold 4','Fold 5','Mean']
observations.T.sort_values(by=['Mean'], ascending = False )
train['text_chr3'] = train['text'].apply(rep_3chr)
 train['selected_text_chr3'] = train['selected_text'].apply(rep_3chr)
 train['selected_text_pred_chr3'] = train['selected_text_pred'].apply(rep_3chr )<filter>
linear_svm = svm.SVC(kernel='linear', C=0.1,gamma=10, probability=True)
 pipe_linear = Pipeline(steps=[('preprocessor', preprocessor),('model', linear_svm)])
 rand = RandomForestClassifier(n_estimators=200, random_state=0)
 pipe_rand = Pipeline(steps=[('preprocessor', preprocessor),('model', rand)])
 log = LogisticRegression()
pipe_log = Pipeline(steps=[('preprocessor', preprocessor),('model', log)])
 ensemble_all = VotingClassifier(estimators=[('Linear_svm', pipe_linear), 
('Random Forest Classifier', pipe_rand), 
('Log', pipe_log)],
voting='soft', weights=[2,3,2] )
train_outlier = train[train['res'] == 0].reset_index(drop=True)
 train_outlier<define_variables>
cross_validation_score = cross_val_score(ensemble_all, train[features], np.ravel(train[target]), scoring='accuracy', cv=cv)
 print("K-Fold scores:", cross_validation_score,
"
Mean:", round(cross_validation_score.mean() , 3),
"
Max:", round(cross_validation_score.max() , 3))
sh_out = str(round(len(train_outlier)*100/len(train),1))
print('Number of outliers is ' + sh_out + '% from training data' )<predict_on_test>
model = RandomForestClassifier(n_estimators=200, random_state = 0)
 pipe = Pipeline(steps=[('preprocessor', preprocessor), 
('model', model)] )
<filter><EOS> 
cross_validation_score_rand = cross_val_score(pipe, train[features], np.ravel(train[target]), scoring='accuracy', cv=cv)
 print("K-Fold scores:", cross_validation_score_rand,
"
Mean:", round(cross_validation_score_rand.mean() , 3),
"
Max:", round(cross_validation_score_rand.max() , 3))