Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<save_to_csv>
warnings.filterwarnings('ignore')
 pd.set_option('precision', 3)
 %config InlineBackend.figure_format = 'retina'
sns.set_style('dark')
 mpl.rcParams['axes.labelsize'] = 14
mpl.rcParams['axes.titlesize'] = 15
mpl.rcParams['xtick.labelsize'] = 12
mpl.rcParams['ytick.labelsize'] = 12
mpl.rcParams['legend.fontsize'] = 12
print('Libraries Loaded!' )
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.head(25 )<define_variables>
train_df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv')
 print('Dataframes loaded!')
 print('Training set: {} rows and {} columns'.format(train_df.shape[0], train_df.shape[1]))
print(' Test set: {} rows and {} columns'.format(test_df.shape[0], test_df.shape[1]))
internet_on = False<install_modules>
all_data = pd.concat([train_df, test_df])
 print('Combined set: {} rows and {} columns'.format(all_data.shape[0], all_data.shape[1]))
print('
Survived?: ')
 all_data['Survived'].value_counts(dropna = False )
if internet_on==True:
!pip install fairseq fastBPE<install_modules>
train_df.drop('PassengerId', axis = 1, inplace = True )
if internet_on==True:
!mv /root/.cache/pip/wheels/df/60/ff/1764bce64cccd9d2c06ba19e5f6f4108ad29e2d48e1068c684/* /kaggle/working
!mv /root/.cache/pip/wheels/fb/85/9b/286072121774d5b8b0253ab66271b558069189cbe795bc6084/* /kaggle/working<install_modules>
men = train_df[train_df['Sex'] == 'male']
women = train_df[train_df['Sex'] == 'female']
if internet_on==False:
!pip install.. /input/fairseq-and-fastbpe/sacrebleu-1.4.9-py3-none-any.whl<install_modules>
train_df['Age_Bin'] =(train_df['Age']//15)*15
train_df[['Age_Bin', 'Survived']].groupby(['Age_Bin'] ).mean()
if internet_on==False:
!pip install.. /input/v1-fairseq-fastbpe/fastBPE-0.1.0-cp36-cp36m-linux_x86_64.whl
!pip install.. /input/v1-fairseq-fastbpe/fairseq-0.9.0-cp36-cp36m-linux_x86_64.whl<import_modules>
print('Number of passengers in each class:')
 train_df['Pclass'].value_counts()
print('TF version',tf.__version__ )<load_pretrained>
train_df['Fare_Bin'] = pd.qcut(train_df['Fare'], 5)
 train_df[['Fare_Bin', 'Survived']].groupby(['Fare_Bin'] ).mean()
class BERTweetTokenizer() :
def __init__(self,pretrained_path = 'pretrained_models/BERTweet_base_transformers/'):
self.bpe = fastBPE(SimpleNamespace(bpe_codes= pretrained_path + "bpe.codes"))
self.vocab = Dictionary()
self.vocab.add_from_file(pretrained_path + "dict.txt")
 self.cls_token_id = 0
self.pad_token_id = 1
self.sep_token_id = 2
self.pad_token = '<pad>'
self.cls_token = '
'
self.sep_token = '
'
def bpe_encode(self,text):
return self.bpe.encode(text)
 def encode(self,text,add_special_tokens=False):
subwords = self.bpe.encode(text)
 input_ids = self.vocab.encode_line(subwords, append_eos=False, add_if_not_exist=False ).long().tolist()
return input_ids
def tokenize(self,text):
return self.bpe_encode(text ).split()
def convert_tokens_to_ids(self,tokens):
input_ids = self.vocab.encode_line(' '.join(tokens), append_eos=False, add_if_not_exist=False ).long().tolist()
return input_ids
def decode_id(self,id):
return self.vocab.string(id, bpe_symbol = '@@')
 def decode_id_nospace(self,id):
return self.vocab.string(id, bpe_symbol = '@@ ' )<define_variables>
alone = train_df[(train_df['SibSp'] == 0)&(train_df['Parch'] == 0)]
not_alone = train_df[(train_df['SibSp'] != 0)|(train_df['Parch'] != 0)]
tokenizer = BERTweetTokenizer('/kaggle/input/bertweet-base-transformers/' )<define_variables>
train_df['Title'] = train_df['Name'].apply(lambda x: x.split(',')[1].split('.')[0].strip())
 train_df['Title'].replace({'Mlle': 'Miss', 'Mme': 'Mrs', 'Ms': 'Miss'}, inplace = True)
 train_df['Title'].replace(['Don', 'Rev', 'Dr', 'Major', 'Lady', 'Sir', 'Col', 'Capt', 'the Countess', 'Jonkheer'],
'Rare Title', inplace = True)
 train_df['Title'].value_counts()
tokenizer.encode('positive' )<define_variables>
print('Cabin:
Number of existing values: ', train_df['Cabin'].notnull().sum())
 print(' Number of unique values: ', train_df['Cabin'].nunique() )
tokenizer.encode('negative' )<set_options>
all_data['Age'] = all_data['Age'].fillna(train_df['Age'].median())
 all_data['Fare'] = all_data['Fare'].fillna(train_df['Fare'].median())
 print('Done!' )
tokenizer.encode('neutral' )<string_transform>
all_data['Last_Name'] = all_data['Name'].apply(lambda x: str.split(x, ',')[0])
 all_data['Fare'].fillna(all_data['Fare'].mean() , inplace = True)
 default_sr_value = 0.5
all_data['Family_Survival'] = default_sr_value
for grp, grp_df in all_data[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId', 'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
all_data.loc[all_data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
all_data.loc[all_data['PassengerId'] == passID, 'Family_Survival'] = 0
for _, grp_df in all_data.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
all_data.loc[all_data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
all_data.loc[all_data['PassengerId'] == passID, 'Family_Survival'] = 0
all_data['Age_Bin'] =(all_data['Age']//15)*15
all_data['Fare_Bin'] = pd.qcut(all_data['Fare'], 5)
 all_data['Relatives'] = all_data['SibSp'] + all_data['Parch']
all_data['Title'] = all_data['Name'].apply(lambda x: x.split(',')[1].split('.')[0].strip())
 all_data['Title'].replace({'Mlle':'Miss', 'Mme':'Mrs', 'Ms':'Miss'}, inplace = True)
 all_data['Title'].replace(['Don', 'Rev', 'Dr', 'Major', 'Lady', 'Sir', 'Col', 'Capt', 'the Countess', 'Jonkheer', 'Dona'],
'Rare Title', inplace = True)
 print('Done!' )
tokenizer.decode_id([14058] )<load_from_csv>
all_data['Fare_Bin'] = LabelEncoder().fit_transform(all_data['Fare_Bin'])
 all_data['Age_Bin'] = LabelEncoder().fit_transform(all_data['Age_Bin'])
 all_data['Title_Bin'] = LabelEncoder().fit_transform(all_data['Title'])
 all_data['Sex'] = LabelEncoder().fit_transform(all_data['Sex'])
 print('Done!' )
def read_train() :
train=pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission()<count_values>
all_data.drop(['PassengerId', 'Age', 'Fare', 'Name', 'SibSp', 'Parch', 'Ticket', 'Cabin', 'Title', 'Last_Name', 'Embarked'], axis = 1, inplace = True)
 print('Done!')
 print('Modified dataset: ')
 all_data.head()
train_df.sentiment.value_counts(dropna=False )<string_transform>
train_df = all_data[:891]
X_train = train_df.drop('Survived', 1)
 y_train = train_df['Survived']
test_df = all_data[891:]
X_test = test_df.copy()
X_test.drop('Survived', axis = 1, inplace = True)
 print('Splitting: Done!' )
def jaccard(str1, str2):
a = set(str(str1 ).lower().split())
 b = set(str(str2 ).lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
std_scaler = StandardScaler()
X_train_scaled = std_scaler.fit_transform(X_train)
 X_test_scaled = std_scaler.transform(X_test)
 print('Scaling: Done!' )
MAX_LEN = 96
PATH = '.. /input/bertweet-base-transformers/'
sentiment_id = {'positive': 1809, 'negative': 3392, 'neutral': 14058}<define_variables>
random_state = 1
clf_list = [GaussianNB() ,
LogisticRegression(random_state = random_state),
KNeighborsClassifier() ,
SVC(random_state = random_state, probability = True),
DecisionTreeClassifier(random_state = random_state),
RandomForestClassifier(random_state = random_state),
XGBClassifier(random_state = random_state),
AdaBoostClassifier(base_estimator = DecisionTreeClassifier(random_state = random_state), random_state = random_state)]
cv_base_mean, cv_std = [], []
for clf in clf_list:
cv = cross_val_score(clf, X_train_scaled, y = y_train, scoring = 'accuracy', cv = 5, n_jobs = -1)
 cv_base_mean.append(cv.mean())
 cv_std.append(cv.std())
 cv_total = pd.DataFrame({'Algorithm': ['Gaussian Naive Bayes', 'Logistic Regression', 'k-Nearest Neighboors', 'SVC', 'Decision Tree', 'Random Forest', 'XGB Classifier', 'AdaBoost Classifier'],
'CV-Means': cv_base_mean,
'CV-Errors': cv_std})
 sns.barplot('CV-Means', 'Algorithm', data = cv_total, palette = 'Paired', orient = 'h', **{'xerr': cv_std})
 plt.xlabel('Mean Accuracy')
 plt.title('Cross Validation Scores')
 plt.xlim([0.725, 0.88])
 plt.axvline(x = 0.80, color = 'firebrick', linestyle = '--');
ct = train_df.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train_df.shape[0]):
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 text2 = " ".join(train_df.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc:
w = tokenizer.decode_id([t])
 if text1[text1.find(w,idx)-1] == " ":
idx+=1
offsets.append(( idx,idx+len(w)))
 idx += len(w)
 else:
offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train_df.loc[k,'sentiment']]
if len(enc)<92:
input_ids[k,:len(enc)+5] = [0] + enc + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1
if len(enc)>91:
input_ids[k,:96] = [0] + enc[:91] + [2,2] + [s_tok] + [2]
attention_mask[k,:96] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,96-1] = 1<categorify>
estimators = [('gnb', clf_list[0]),('lr', clf_list[1]), 
('knn', clf_list[2]),('svc', clf_list[3]), 
('dt', clf_list[4]),('rf', clf_list[5]), 
('xgb', clf_list[6]),('ada', clf_list[7])]
base_voting_hard = VotingClassifier(estimators = estimators , voting = 'hard')
 base_voting_soft = VotingClassifier(estimators = estimators , voting = 'soft')
 cv_hard = cross_val_score(base_voting_hard, X_train_scaled, y_train, cv = 5)
 cv_soft = cross_val_score(base_voting_soft, X_train_scaled, y_train, cv = 5)
 print('Baseline Models - Ensemble
--------------------------')
 print('Hard Voting: {}%'.format(np.round(cv_hard.mean() *100, 1)))
 print('Soft Voting: {}%'.format(np.round(cv_soft.mean() *100, 1)) )
ct = test_df.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test_df.shape[0]):
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test_df.loc[k,'sentiment']]
if len(enc)<92:
input_ids_t[k,:len(enc)+5] = [0] + enc + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc)+5] = 1
if len(enc)>91:
input_ids_t[k,:96] = [0] + enc[:91] + [2,2] + [s_tok] + [2]
attention_mask_t[k,:96] = 1<categorify>
base_voting_hard.fit(X_train_scaled, y_train)
 base_voting_soft.fit(X_train_scaled, y_train)
 y_pred_base_hard = base_voting_hard.predict(X_test_scaled)
 y_pred_base_soft = base_voting_hard.predict(X_test_scaled )
all=[]
count=0
for k in range(train_df.shape[0]):
a = np.argmax(start_tokens[k,])
 b = np.argmax(end_tokens[k,])
 text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:b])
 st = st.replace('<unk>','')
 all.append(jaccard(st,train_df.loc[k,'selected_text']))
print('>>>> Jaccard =',np.mean(all))<define_variables>
cv_means_tuned = [np.nan]
def clf_performance(classifier, model_name):
print(model_name)
 print('-------------------------------')
 print(' Best Score: ' + str(classifier.best_score_))
print(' Best Parameters: ' + str(classifier.best_params_))
cv_means_tuned.append(classifier.best_score_ )
improve_jacc_review = False<string_transform>
lr = LogisticRegression()
param_grid = {'max_iter' : [100],
'penalty' : ['l1', 'l2'],
'C' : np.logspace(-2, 2, 20),
'solver' : ['lbfgs', 'liblinear']}
clf_lr = GridSearchCV(lr, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)
 best_clf_lr = clf_lr.fit(X_train_scaled, y_train)
 clf_performance(best_clf_lr, 'Logistic Regression' )
if improve_jacc_review == True:
all=[]
count=0
for k in range(train_df.shape[0]):
a = np.argmax(start_tokens[k,])
 b = np.argmax(end_tokens[k,])
 text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:b])
 st = st.replace('<unk>','')
 if jaccard(st,train_df.loc[k,'selected_text'])<.3:
print(k)
 print(st)
 print(train_df.loc[k,'selected_text'])
 print()
all.append(jaccard(st,train_df.loc[k,'selected_text']))
print('>>>> Jaccard =',np.mean(all))<choose_model_class>
knn = KNeighborsClassifier()
param_grid = {'n_neighbors' : np.arange(3, 30, 2),
'weights': ['uniform', 'distance'],
'algorithm': ['auto'],
'p': [1, 2]}
clf_knn = GridSearchCV(knn, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)
 best_clf_knn = clf_knn.fit(X_train_scaled, y_train)
 clf_performance(best_clf_knn, 'KNN' )
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'model.bin',config=config,from_pt=True)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Conv1D(128, 2,padding='same' )(x[0])
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Conv1D(128, 2, padding='same' )(x[0])
 x2 = tf.keras.layers.BatchNormalization()(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=5e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 return model<define_variables>
svc = SVC(probability = True)
 param_grid = tuned_parameters = [{'kernel': ['rbf'],
'gamma': [0.01, 0.1, 0.5, 1, 2, 5],
'C': [.1, 1, 2, 5]},
{'kernel': ['linear'],
'C': [.1, 1, 2, 10]},
{'kernel': ['poly'],
'degree' : [2, 3, 4, 5],
'C': [.1, 1, 10]}]
clf_svc = GridSearchCV(svc, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)
 best_clf_svc = clf_svc.fit(X_train_scaled, y_train)
 clf_performance(best_clf_svc, 'SVC' )
n_splits = 5<define_variables>
dt = DecisionTreeClassifier(random_state = 1)
 param_grid = {'max_depth': [3, 5, 10, 20, 50],
'criterion': ['entropy', 'gini'],
'min_samples_split': [5, 10, 15, 30],
'max_features': [None, 'auto', 'sqrt', 'log2']}
clf_dt = GridSearchCV(dt, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)
 best_clf_dt = clf_dt.fit(X_train_scaled, y_train)
 clf_performance(best_clf_dt, 'Decision Tree' )
n_seeds = 1<train_model>
rf = RandomForestClassifier(random_state = 42)
 param_grid = {'n_estimators': [50, 150, 300, 450],
'criterion': ['entropy'],
'bootstrap': [True],
'max_depth': [3, 5, 10],
'max_features': ['auto','sqrt'],
'min_samples_leaf': [2, 3],
'min_samples_split': [2, 3]}
clf_rf = GridSearchCV(rf, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)
 best_clf_rf = clf_rf.fit(X_train_scaled, y_train)
 clf_performance(best_clf_rf, 'Random Forest' )
trainModel=False<define_variables>
xgb = XGBClassifier(random_state = 42)
 param_grid = {'n_estimators': [15, 25, 50, 100],
'colsample_bytree': [0.65, 0.75, 0.80],
'max_depth': [None],
'reg_alpha': [1],
'reg_lambda': [1, 2, 5],
'subsample': [0.50, 0.75, 1.00],
'learning_rate': [0.01, 0.1, 0.5],
'gamma': [0.5, 1, 2, 5],
'min_child_weight': [0.01],
'sampling_method': ['uniform']}
clf_xgb = GridSearchCV(xgb, param_grid = param_grid, cv = 3, verbose = True, n_jobs = -1)
 best_clf_xgb = clf_xgb.fit(X_train_scaled, y_train)
 clf_performance(best_clf_xgb, 'XGB' )
if trainModel==True:
for x in range(n_seeds):
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=n_splits,shuffle=True,random_state=777+x)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train_df.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
reduce_lr = tf.keras.callbacks.LearningRateScheduler(scheduler)
 sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i-%x.h5'%(VER,fold,x), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 hist = model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=3, batch_size=8, verbose=DISPLAY, callbacks=[sv, reduce_lr],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('%s-roberta-%i-%x.h5'%(VER,fold,x))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/(n_splits*n_seeds)
 preds_end += preds[1]/(n_splits*n_seeds)
 all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:a+3])
 else:
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:b])
 st = st.replace('<unk>','')
 all.append(jaccard(st,train_df.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))<define_variables>
adaDTC = AdaBoostClassifier(base_estimator = DecisionTreeClassifier(random_state = random_state), random_state=random_state)
 param_grid = {'algorithm': ['SAMME', 'SAMME.R'],
'base_estimator__criterion' : ['gini', 'entropy'],
'base_estimator__splitter' : ['best', 'random'],
'n_estimators': [2, 5, 10, 50],
'learning_rate': [0.01, 0.1, 0.2, 0.3, 1, 2]}
clf_ada = GridSearchCV(adaDTC, param_grid = param_grid, cv = 5, scoring = 'accuracy', n_jobs = -1, verbose = 1)
 best_clf_ada = clf_ada.fit(X_train_scaled, y_train)
 clf_performance(best_clf_ada, 'AdaBost' )
if trainModel==False:
DISPLAY=1
for x in range(n_seeds):
jac = []
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
print('
print('
print('
skf = StratifiedKFold(n_splits=n_splits,shuffle=True,random_state=777+x)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train_df.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
model.load_weights('.. /input/bertweet-files/v0-roberta-%i-%x.h5'%(fold,x))
print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/(n_splits*n_seeds)
 preds_end += preds[1]/(n_splits*n_seeds)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 all = []; pos = []; neg = []; nue = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:a+3])
 else:
text1 = " "+" ".join(train_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:b])
 st = st.replace('<unk>','')
 if train_df.loc[k,'sentiment']=='positive':
pos.append(jaccard(st,train_df.loc[k,'selected_text']))
if train_df.loc[k,'sentiment']=='negative':
neg.append(jaccard(st,train_df.loc[k,'selected_text']))
if train_df.loc[k,'sentiment']=='neutral':
st = text1
nue.append(jaccard(st,train_df.loc[k,'selected_text']))
all.append(jaccard(st,train_df.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print('>>>> FOLD %i Neutral Jaccard ='%(fold+1),np.mean(nue))
print('>>>> FOLD %i Positive Jaccard ='%(fold+1),np.mean(pos))
print('>>>> FOLD %i Negative Jaccard ='%(fold+1),np.mean(neg))
print()
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))<categorify>
cv_total = pd.DataFrame({'Algorithm': ['Gaussian Naive Bayes', 'Logistic Regression', 'k-Nearest Neighboors', 'SVC', 'Decision Tree', 'Random Forest', 'XGB Classifier', 'AdaBoost Classifier'],
'Baseline': cv_base_mean,
'Tuned Performance': cv_means_tuned})
 cv_total
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:a+3])
 else:
text1 = " "+" ".join(test_df.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode_id_nospace(enc[a-1:b])
 st = st.replace('<unk>','')
 if test_df.loc[k,'sentiment']=='neutral':
st = text1
all.append(st )<save_to_csv>
best_lr = best_clf_lr.best_estimator_
best_knn = best_clf_knn.best_estimator_
best_svc = best_clf_svc.best_estimator_
best_dt = best_clf_dt.best_estimator_
best_rf = best_clf_rf.best_estimator_
best_xgb = best_clf_xgb.best_estimator_
estimators = [('lr', best_lr),('knn', best_knn),('svc', best_svc), 
('rf', best_rf),('xgb', best_xgb),('dt', best_dt)]
tuned_voting_hard = VotingClassifier(estimators = estimators, voting = 'hard', n_jobs = -1)
 tuned_voting_soft = VotingClassifier(estimators = estimators, voting = 'soft', n_jobs = -1)
 tuned_voting_hard.fit(X_train_scaled, y_train)
 tuned_voting_soft.fit(X_train_scaled, y_train)
 cv_hard = cross_val_score(tuned_voting_hard, X_train_scaled, y_train, cv = 5)
 cv_soft = cross_val_score(tuned_voting_soft, X_train_scaled, y_train, cv = 5)
 print('Tuned Models - Ensemble
-----------------------')
 print('Hard Voting: {}%'.format(np.round(cv_hard.mean() *100, 2)))
 print('Soft Voting: {}%'.format(np.round(cv_soft.mean() *100, 2)))
 y_pred_tuned_hd = tuned_voting_hard.predict(X_test_scaled ).astype(int)
 y_pred_tuned_sf = tuned_voting_soft.predict(X_test_scaled ).astype(int )
<import_modules><EOS> 
test_df = pd.DataFrame(pd.read_csv('.. /input/titanic/test.csv')['PassengerId'])
 pd.DataFrame(data = {'PassengerId': test_df.PassengerId,
'Survived': y_pred_base_hard.astype(int)} ).to_csv('01-Baseline_Hard_voting.csv', index = False)
 pd.DataFrame(data = {'PassengerId': test_df.PassengerId,
'Survived': y_pred_base_soft.astype(int)} ).to_csv('02-Baseline_Soft_voting.csv', index = False)
 pd.DataFrame(data = {'PassengerId': test_df.PassengerId,
'Survived': y_pred_tuned_hd.astype(int)} ).to_csv('03-Tuned_Hard_Voting.csv', index = False)
 pd.DataFrame(data = {'PassengerId': test_df.PassengerId,
'Survived': y_pred_tuned_sf.astype(int)} ).to_csv('04-Tuned_Soft_Voting.csv', index = False )