Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<filter>
rnd_state = 42
skfold = StratifiedKFold(n_splits=5 )
train_outlier[train_outlier['share']==1]<filter>
train_data = pd.read_csv(".. /input/train.csv")
 test_data = pd.read_csv(".. /input/test.csv")
 train_data.head()
train_good[train_good['share']==1]<filter>
train_data["Age_categories"] = np.ceil(train_data["Age"]/1.5)
 train_data["Age_categories"].where(train_data["Age_categories"] <2, 2.0, inplace=True)
 train_data["Age_categories"].where(train_data["Age_categories"] >50, 50.0, inplace=True)
 split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state= rnd_state)
 for train_index, test_index in split.split(train_data, train_data["Age_categories"]):
strat_train_set = train_data.loc[train_index]
strat_test_set = train_data.loc[test_index]
for set_ in(strat_train_set, strat_test_set):
set_.drop("Age_categories", axis=1, inplace=True)
 strat_train_set_p = strat_train_set.drop("Survived", axis=1)
 survival_labels = strat_train_set["Survived"].copy()
strat_test_set_p = strat_test_set.drop("Survived", axis=1)
 validation_labels = strat_test_set["Survived"].copy()
len(train_outlier[train_outlier["text"].str.find(' ')== -1] )<filter>
def extract_title(dataset):
dataset['Title'] = dataset.Name.apply(lambda name: name.split(',')[1].split('.')[0].strip())
 normalized_titles = {
"Capt": "other",
"Col": "other",
"Major": "other",
"Jonkheer": "other",
"Don": "Mr",
"Sir" : "Mr",
"Dr": "other",
"Rev": "other",
"the Countess":"other",
"Dona": "Mrs",
"Mme": "Mrs",
"Mlle": "Miss",
"Ms": "Mrs",
"Mr" : "Mr",
"Mrs" : "Mrs",
"Miss" : "Miss",
"Master" : "Master",
"Lady" : "Mrs",
"NaN" : "other"
}
dataset.Title = dataset.Title.map(normalized_titles)
 return dataset
len(train_good[train_good["text"].str.find(' ')== -1] )<feature_engineering>
def data_cleaner(dataset):
dataset = extract_title(dataset)
 return dataset
def subtext_analysis(col, subtext, df1, str1, df2, str2):
result = pd.DataFrame(columns = ['subtext', str1, str2, 'share,%'])
 if(len(df1)> 0)and(len(df2)> 0):
for i in range(len(subtext)) :
result.loc[i,'subtext'] = subtext[i]
num1 = len(df1[df1[col].str.find(subtext[i])> -1])
 result.loc[i, str1] = num1
num2 = len(df2[df2[col].str.find(subtext[i])> -1])
 result.loc[i, str2] = num2
result.loc[i,'share,%'] = round(num1*100/num2,1)if num2 != 0 else 0
print('Number of all data is', len(df2))
display(result.sort_values(by=['share,%', str1], ascending=False))<feature_engineering>
class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
def __init__(self, add_familySize = False):
self.add_familySize = add_familySize
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
familySize = X[:, Sibsp_ix] + X[:, Parch_ix]
if self.add_familySize:
return np.c_[X, familySize]
else:
return np.c_[X]
Sibsp_ix, Parch_ix = 2, 3
numeric_features = ['Age','Fare', 'SibSp', 'Parch']
numeric_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='median')) , 
('attribs_adder', CombinedAttributesAdder(add_familySize = False)) , 
('scaler', StandardScaler())
 ])
 categorical_features = ['Title', 'Sex', 'Pclass', 'Embarked']
categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='constant', fill_value='S')) , 
('onehot', OneHotEncoder(handle_unknown='ignore', sparse=False)) ])
 preprocessor = ColumnTransformer(
 transformers=[ 
('num', numeric_transformer, numeric_features), 
('cat', categorical_transformer, categorical_features)
 ] )
def subtext_analysis_one_df(col, subtext, df, str):
result = pd.DataFrame(columns = ['subtext', str, 'share of all,%'])
 num_all = len(df)
 if(num_all > 0):
for i in range(len(subtext)) :
result.loc[i,'subtext'] = subtext[i]
num = len(df[df[col].str.find(subtext[i])> -1])
 result.loc[i, str] = num
result.loc[i,'share of all,%'] = round(num*100/num_all,1)
 print('Number of all data is', len(df))
display(result.sort_values(by='share of all,%', ascending=False))<define_variables>
train_set_prepared = preprocessor.fit_transform(data_cleaner(strat_train_set_p))
validation_set_prepared = preprocessor.fit_transform(data_cleaner(strat_test_set_p))
subtext_test = ['SAD', 'bullying', 'Uh', 'oh', 'onna', 'fun', 'addicted', 'Power', 'well', 'unhappy', 'funny', 'Tears', 'Fears', 'sleeeeepy', ' ', ',', '?', '!' ,'!!', '!!!', ':/', '...', 'http', '****']<compute_test_metric>
log_clf = LogisticRegression(random_state=rnd_state, solver='lbfgs')
 log_clf.fit(train_set_prepared, survival_labels )
subtext_analysis("selected_text", subtext_test, train_outlier, 'train_outliers', train, 'train_all' )<compute_test_metric>
log_pred = log_clf.predict(validation_set_prepared ).astype(int )
subtext_analysis("selected_text", subtext_test, train_good, 'train_good', train, 'train_all' )<compute_test_metric>
log_cross_scores = cross_val_score(log_clf, train_set_prepared, survival_labels, cv=skfold, scoring="accuracy")
 log_scores =(log_cross_scores.mean() + accuracy_score(log_pred, validation_labels)) /2
log_scores
subtext_analysis_one_df("selected_text", subtext_test, train, 'test_all' )<compute_test_metric>
sgd_clf = SGDClassifier(max_iter=60, penalty = None, eta0=0.1, random_state=rnd_state, tol =1e-3)
 sgd_clf.fit(train_set_prepared, survival_labels )
subtext_analysis_one_df("selected_text", subtext_test, test, 'test_all' )<feature_engineering>
sgd_cross_scores = cross_val_score(sgd_clf, train_set_prepared, survival_labels, cv=skfold, scoring="accuracy")
 sgd_pred = sgd_clf.predict(validation_set_prepared ).astype(int)
 sgd_scores =(sgd_cross_scores.mean() + accuracy_score(sgd_pred, validation_labels)) /2
sgd_scores
test['text_chr3'] = test['text'].apply(rep_3chr)
 test.head(10 )<train_model>
svm_clf = SVC(gamma='auto')
 svm_clf.fit(train_set_prepared, survival_labels )
train_outlier1 = train_outlier.nsmallest(1000, 'metric', keep='all')
 train_outlier2 = train_outlier.nsmallest(2000, 'metric', keep='all')
 train_outlier3 = train_outlier.nsmallest(3000, 'metric', keep='all')
 train_outlier5 = train_outlier.nsmallest(5000, 'metric', keep='all')
 train_outlier8 = train_outlier.nsmallest(8000, 'metric', keep='all' )<compute_test_metric>
svm_cross_scores = cross_val_score(svm_clf, train_set_prepared, survival_labels, cv=skfold, scoring="accuracy")
 svm_pred = svm_clf.predict(validation_set_prepared ).astype(int)
 svm_scores =(svm_cross_scores.mean() + accuracy_score(svm_pred, validation_labels)) /2
svm_scores
subtext_analysis("selected_text", subtext_test, train_outlier1, 'in worst 1000 outliers', train_outlier, 'in all outliers' )<compute_test_metric>
subtext_analysis("selected_text", subtext_test, train_outlier2, 'in worst 2000 outliers', train_outlier, 'in all outliers' )<compute_test_metric>
svm_grid_clf = SVC(kernel="rbf", gamma=0.1, C=10)
 svm_grid_clf.fit(train_set_prepared, survival_labels)
 svm_grid_cross_scores = cross_val_score(svm_grid_clf, train_set_prepared, survival_labels, cv=skfold, scoring="accuracy")
 svm_pred = svm_grid_clf.predict(validation_set_prepared ).astype(int)
 svm_grid_scores =(svm_grid_cross_scores.mean() + accuracy_score(svm_pred, validation_labels)) /2
svm_grid_scores
subtext_analysis("selected_text", subtext_test, train_outlier3, 'in worst 3000 outliers', train_outlier, 'in all outliers' )<compute_test_metric>
forest_clf = RandomForestClassifier(random_state=rnd_state, n_estimators=10)
 forest_clf.fit(train_set_prepared, survival_labels )
subtext_analysis("selected_text", subtext_test, train_outlier5, 'in worst 5000 outliers', train_outlier, 'in all outliers' )<compute_test_metric>
param_grid = [
{'n_estimators': [200, 250], 'max_features': [10, 17]},
{'bootstrap': [False], 'n_estimators': [200, 250], 'max_features': [10, 12, 17]},
]
grid_forest_clf = GridSearchCV(forest_clf, param_grid, cv=skfold,
scoring='accuracy', return_train_score=True)
 grid_forest_clf.fit(train_set_prepared, survival_labels )
subtext_analysis("selected_text", subtext_test, train_outlier8, 'in worst 8000 outliers', train_outlier, 'in all outliers' )<define_variables>
grid_forest_clf.best_params_
col_hist = ['sentiment', 'start', 'end', 'start_pred', 'end_pred', 'len_text', 'len_selected_text',
'text_chr3', 'selected_text_chr3', 'selected_text_pred_chr3', 'metric']<drop_column>
grid_forest_pred = grid_forest_clf.predict(validation_set_prepared ).astype(int)
 grid_forest_scores =(grid_forest_clf.best_score_ + accuracy_score(grid_forest_pred, validation_labels)) /2
grid_forest_scores
data = train[['sentiment', 'start', 'end', 'start_pred', 'end_pred', 'len_text', 'len_selected_text', 'diff_num', 'share', 'metric', 'res']].dropna()
data<set_options>
param_distribs = {
'n_estimators': randint(low=100, high=300),
'max_features': randint(low=8, high=17),
}
rnd_forest_clf = RandomizedSearchCV(forest_clf, param_distributions=param_distribs,
n_iter=10, cv=skfold, scoring='accuracy', random_state=rnd_state)
 rnd_forest_clf.fit(train_set_prepared, survival_labels )
%matplotlib inline
pd.set_option('max_colwidth', 40 )<define_variables>
rnd_forest_clf.best_params_
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
SEED = 88888
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<feature_engineering>
rnd_forest_pred = rnd_forest_clf.predict(validation_set_prepared ).astype(int)
 rnd_forest_scores =(rnd_forest_clf.best_score_ + accuracy_score(rnd_forest_pred, validation_labels)) /2
rnd_forest_scores
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1<init_hyperparams>
knn_clf = KNeighborsClassifier()
knn_clf.fit(train_set_prepared, survival_labels)
 knn_cross_scores = cross_val_score(knn_clf, train_set_prepared, survival_labels, cv=skfold, scoring="accuracy")
 knn_pred = knn_clf.predict(validation_set_prepared ).astype(int)
 knn_scores =(knn_cross_scores.mean() + accuracy_score(knn_pred, validation_labels)) /2
knn_scores
Dropout_new = 0.15
n_split = 5
lr = 3e-5<train_model>
param_grid = [{'weights': ["uniform", "distance"], 'n_neighbors': [3, 4, 5]}]
grid_knn_clf = GridSearchCV(knn_clf, param_grid, cv=5, verbose=3, n_jobs=-1)
 grid_knn_clf.fit(train_set_prepared, survival_labels)
 grid_knn_clf.best_params_
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(Dropout_new )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Conv1D(64, 2,padding='same' )(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(Dropout_new )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Conv1D(64, 2, padding='same' )(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=lr)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model<string_transform>
grid_knn_cross_scores = cross_val_score(grid_knn_clf, train_set_prepared, survival_labels, cv=skfold, scoring="accuracy")
 grid_knn_pred = grid_knn_clf.predict(validation_set_prepared ).astype(int)
 grid_knn_scores =(grid_knn_cross_scores.mean() + accuracy_score(grid_knn_pred, validation_labels)) /2
grid_knn_scores
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
ext_clf = ExtraTreesClassifier()
param_grid = {"max_depth": [None],
"max_features": [10, 17],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False, True],
"n_estimators" :[50,100,200],
"criterion": ["gini"]}
grid_ext_clf = GridSearchCV(ext_clf,param_grid, cv=skfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 grid_ext_clf.fit(train_set_prepared, survival_labels)
 grid_ext_clf.best_params_
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1<define_variables>
grid_ext_pred = grid_ext_clf.predict(validation_set_prepared ).astype(int)
 grid_ext_scores =(grid_ext_clf.best_score_ + accuracy_score(grid_ext_pred, validation_labels)) /2
grid_ext_scores
%%time
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_end_train = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=n_split,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting all Train for Outlier analysis...')
 preds_train = padded_model.predict([input_ids,attention_mask,token_type_ids],verbose=DISPLAY)
 preds_start_train += preds_train[0]/skf.n_splits
preds_end_train += preds_train[1]/skf.n_splits
print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<compute_test_metric>
ada_clf = AdaBoostClassifier(
 DecisionTreeClassifier(random_state=rnd_state, max_depth=2),
random_state = rnd_state)
 param_grid = {"base_estimator__criterion" : ["gini", "entropy"],
"base_estimator__splitter" : ["best", "random"],
"algorithm" : ["SAMME","SAMME.R"],
"n_estimators" :[10, 100, 200, 250],
"learning_rate": [0.05, 0.5, 1.5, 2.5]}
grid_ada_clf = GridSearchCV(ada_clf, param_grid, cv=skfold, scoring="accuracy", n_jobs= -1, verbose = 1)
 grid_ada_clf.fit(train_set_prepared, survival_labels)
 grid_ada_clf.best_params_
ada_pred = grid_ada_clf.predict(validation_set_prepared ).astype(int)
 grid_ada_scores =(grid_ada_clf.best_score_ + accuracy_score(ada_pred, validation_labels)) /2
grid_ada_scores
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))
print(jac )<string_transform>
gb_clf = GradientBoostingClassifier(random_state=rnd_state)
 param_grid = {
'n_estimators' : [25, 50 ,75, 100, 200],
'learning_rate': [0.005 ,0.05, 0.5, 1.5],
'max_depth': [2, 4, 6, 8],
'max_features': [10, 12, 17]
}
grid_gb_clf = GridSearchCV(gb_clf, param_grid, cv=skfold, scoring="accuracy", n_jobs= -1, verbose = 1)
 grid_gb_clf.fit(train_set_prepared, survival_labels)
 gb_pred = grid_gb_clf.predict(validation_set_prepared ).astype(int)
 grid_gb_scores =(grid_gb_clf.best_score_ + accuracy_score(gb_pred, validation_labels)) /2
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(st )<save_to_csv>
grid_gb_clf.best_params_
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 test.sample(10 )<define_variables>
voting_clf = VotingClassifier(
 estimators=[('rf', rnd_forest_clf),('svc', svm_clf),('knn', grid_knn_clf)],
voting='hard')
 voting_clf.fit(train_set_prepared, survival_labels )
all = []
start = []
end = []
start_pred = []
end_pred = []
for k in range(input_ids.shape[0]):
a = np.argmax(preds_start_train[k,])
 b = np.argmax(preds_end_train[k,])
 start.append(np.argmax(start_tokens[k]))
end.append(np.argmax(end_tokens[k]))
if a>b:
st = train.loc[k,'text']
start_pred.append(0)
 end_pred.append(len(st))
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 start_pred.append(a)
 end_pred.append(b)
 all.append(st)
 train['start'] = start
train['end'] = end
train['start_pred'] = start_pred
train['end_pred'] = end_pred
train['selected_text_pred'] = all
train.sample(10 )<compute_train_metric>
voting_cross_scores = cross_val_score(voting_clf, train_set_prepared, survival_labels, cv=skfold, scoring="accuracy")
 voting_pred = voting_clf.predict(validation_set_prepared ).astype(int)
 voting_scores =(voting_cross_scores.mean() + accuracy_score(voting_pred, validation_labels)) /2
voting_scores
def metric_tse(df,col1,col2):
return df.apply(lambda x: jaccard(x[col1],x[col2]),axis=1 )<feature_engineering>
test_set_prepared = preprocessor.fit_transform(data_cleaner(test_data))
predictions = voting_clf.predict(test_set_prepared ).astype(int )
<feature_engineering><EOS> 
ids = test_data['PassengerId']
output = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 output.to_csv('submission.csv', index=False )