Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<install_modules>
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
!mkdir -p data
!pip install '/kaggle/input/simple-transformers-pypi/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/simple-transformers-pypi/simpletransformers-0.22.1-py3-none-any.whl' -q
use_cuda = True
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1
def do_qa_train(train):
output = {}
output['version'] = 'v1.0'
output['data'] = []
paragraphs = []
for line in train:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer.lower() })
 break
qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
def do_qa_test(test):
paragraphs = []
for line in test:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
train_df = read_train()
test_df = read_test()
submission_df_albert = read_submission()
train = np.array(train_df)
 test = np.array(test_df)
 qa_train = do_qa_train(train)
 with open('data/train.json', 'w')as outfile:
json.dump(qa_train, outfile)
 output = {}
output['version'] = 'v1.0'
output['data'] = []
qa_test = do_qa_test(test)
 with open('data/test.json', 'w')as outfile:
json.dump(qa_test, outfile)
 MODEL_PATH = '/kaggle/input/pretrained-albert-pytorch/albert-base-v2/'
model = QuestionAnsweringModel('albert',
MODEL_PATH,
args={"reprocess_input_data": True,
"overwrite_output_dir": True,
"learning_rate": 8e-05,
"num_train_epochs": 3,
"max_seq_length": 192,
"weight_decay": 0.001,
"doc_stride": 64,
"save_model_every_epoch": False,
"fp16": False,
"do_lower_case": True,
'max_query_length': 8,
'max_answer_length': 150
},
use_cuda=use_cuda)
 model.train_model('data/train.json')
 predictions = model.predict(qa_test)
 predictions_df = pd.DataFrame.from_dict(predictions)
 submission_df_albert['selected_text'] = predictions_df['answer']
submission_df_albert = post_process(submission_df_albert, test_df)
<install_modules>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
!mkdir -p data
!pip install '/kaggle/input/simple-transformers-pypi/seqeval-0.0.12-py3-none-any.whl' -q
!pip install '/kaggle/input/simple-transformers-pypi/simpletransformers-0.22.1-py3-none-any.whl' -q
use_cuda = True
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1
def do_qa_train(train):
output = {}
output['version'] = 'v1.0'
output['data'] = []
paragraphs = []
for line in train:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer.lower() })
 break
qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
def do_qa_test(test):
paragraphs = []
for line in test:
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context.lower() , 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 return paragraphs
train_df = read_train()
test_df = read_test()
submission_df_bert = read_submission()
train = np.array(train_df)
 test = np.array(test_df)
 qa_train = do_qa_train(train)
 with open('data/train.json', 'w')as outfile:
json.dump(qa_train, outfile)
 output = {}
output['version'] = 'v1.0'
output['data'] = []
qa_test = do_qa_test(test)
 with open('data/test.json', 'w')as outfile:
json.dump(qa_test, outfile)
 MODEL_PATH = '/kaggle/input/bert-base-uncased/'
model = QuestionAnsweringModel('bert',
MODEL_PATH,
args={"reprocess_input_data": True,
"overwrite_output_dir": True,
"learning_rate": 8e-05,
"num_train_epochs": 3,
"max_seq_length": 192,
"weight_decay": 0.001,
"doc_stride": 64,
"save_model_every_epoch": False,
"fp16": False,
"do_lower_case": True,
'max_query_length': 8,
'max_answer_length': 150
},
use_cuda=use_cuda)
 model.train_model('data/train.json')
 predictions = model.predict(qa_test)
 predictions_df = pd.DataFrame.from_dict(predictions)
 submission_df_bert['selected_text'] = predictions_df['answer']
submission_df_bert = post_process(submission_df_bert, test_df)
<merge>
train_ID = train['PassengerId']
test_ID = test['PassengerId']
train = train.drop('PassengerId', axis=1)
 test = test.drop('PassengerId', axis=1 )
sentiment_analyzer = SentimentIntensityAnalyzer()
def merge_predictions(submission_distil, sumbission_albert, submission_bert):
def merge(row):
pred_1 = set(str(row[1] ).split())
 pred_2 = set(str(row[2] ).split())
 pred_3 = set(str(row[3] ).split())
 res = []
res = list(pred_1.intersection(pred_2, pred_3))
if len(res)== 0:
res = list(pred_1.union(pred_2, pred_3))
return " ".join(res)
 def merge_compound(row):
pred_1 = str(row[1])
 pred_2 = str(row[2])
 pred_3 = str(row[3])
 preds = [pred_1, pred_2, pred_3]
comp_1 = abs(sentiment_analyzer.polarity_scores(pred_1 ).get("compound",0))
comp_2 = abs(sentiment_analyzer.polarity_scores(pred_2 ).get("compound",0))
comp_3 = abs(sentiment_analyzer.polarity_scores(pred_3 ).get("compound",0))
comps = [comp_1, comp_2, comp_3]
best_comp = comps.index(max(comps))
return preds[best_comp]
def capitalize_selected(df):
rows = df.to_dict("records")
 new_rows = []
for r in rows:
text = r["text"]
lower_text = text.lower()
pred_distil = r["selected_text_distil"]
pred_albert = r["selected_text_albert"]
pred_bert = r["selected_text_bert"]
new_pred_distil = text[lower_text.find(pred_distil):len(pred_distil)]
new_pred_albert = text[lower_text.find(pred_albert):len(pred_albert)]
new_pred_bert = text[lower_text.find(pred_bert):len(pred_bert)]
new_row = {
"textID": r['textID'],
"selected_text_distil": new_pred_distil,
"selected_text_albert": new_pred_albert,
"selected_text_bert": new_pred_bert
}
new_rows.append(new_row)
 return pd.DataFrame(new_rows)
 sumbission_albert = sumbission_albert.rename({'selected_text': 'selected_text_albert'}, axis=1)
 del sumbission_albert["textID"]
submission_distil = submission_distil.rename({'selected_text': 'selected_text_distil'}, axis=1)
 submission_bert = submission_bert.rename({'selected_text': 'selected_text_bert'}, axis=1)
 del submission_bert["textID"]
final_sub = pd.concat([submission_distil, sumbission_albert, submission_bert], axis=1)
 final_sub["selected_text"] = final_sub.apply(lambda x: merge_compound(x), axis=1)
 del final_sub["selected_text_albert"]
del final_sub["selected_text_distil"]
del final_sub["selected_text_bert"]
return final_sub

 submission = merge_predictions(submission_df_distil, submission_df_albert, submission_df_bert)
 submission.to_csv('submission.csv', index=False)
<import_modules>
train['Sex'].value_counts()
print('TF version',tf.__version__ )<define_variables>
train['Embarked'].value_counts()
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<define_variables>
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.Survived.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['Survived'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<feature_engineering>
total = all_data.isnull().sum().sort_values(ascending=False)
 percent =(all_data.isnull().sum() /all_data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
all_data.drop(['Cabin'], axis=1, inplace=True )
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model<string_transform>
all_data['SibSp'].loc[np.isnan(all_data['Age'])].value_counts()
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
all_data["Age"] = all_data.loc[all_data["SibSp"]>1].groupby("SibSp")["Age"].transform(
 lambda x: x.fillna(x.median()))
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5'%(VER,fold), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=3, batch_size=32, verbose=DISPLAY, callbacks=[sv],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('%s-roberta-%i.h5'%(VER,fold))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<feature_engineering>
all_data["Age"] = all_data.groupby("Pclass")["Age"].transform(
 lambda x: x.fillna(x.median()))
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))<string_transform>
all_data["Embarked"].value_counts()
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )<save_to_csv>
all_data["Embarked"] = all_data["Embarked"].fillna("S" )
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )<import_modules>
all_data["Fare"] = all_data.groupby("Pclass")["Fare"].transform(
 lambda x: x.fillna(x.median()))
print('TF version',tf.__version__ )<define_variables>
all_data["TotalRelatives"] = all_data['SibSp'] + all_data['Parch']
all_data['IsAlone'] = 1
all_data['IsAlone'].loc[all_data["TotalRelatives"] > 0] = 0
all_data['Title'] = all_data['Name'].str.split(", ", expand=True)[1].str.split(".", expand=True)[0]
all_data['FareBin'] = pd.qcut(all_data['Fare'], 4)
 all_data['AgeBin'] = pd.cut(all_data['Age'].astype(int), 5 )
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<define_variables>
final_features = pd.get_dummies(all_data ).reset_index(drop=True)
 final_features.shape
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<feature_engineering>
train = final_features[:ntrain]
test = final_features[ntrain:]
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
n_folds = 5
def score_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 score = cross_val_score(model, train.values, y_train, cv = kf)
 return("score: {:.4f}({:.4f})".format(score.mean() , score.std()))
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=0.0001)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model<string_transform>
params = {'logisticregression__C' : [0.001,0.01,0.1,1,10,100,1000]}
pipe = make_pipeline(RobustScaler() , LogisticRegression())
 gridsearch_logistic = GridSearchCV(pipe, params, cv=10)
 gridsearch_logistic.fit(train, y_train)
 print("Meilleurs parametres: ", gridsearch_logistic.best_params_ )
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
score_cv(gridsearch_logistic.best_estimator_ )
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5'%(VER,fold), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=5, batch_size=32, verbose=DISPLAY, callbacks=[sv],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('%s-roberta-%i.h5'%(VER,fold))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<feature_engineering>
params = {'kneighborsclassifier__n_neighbors' : [3,4,5,6,7],
'kneighborsclassifier__weights' : ['uniform','distance'],
'kneighborsclassifier__algorithm' : ['auto', 'ball_tree', 'kd_tree', 'brute']}
pipe = make_pipeline(RobustScaler() , KNeighborsClassifier())
 gridsearch_KNC = GridSearchCV(pipe, params, cv=5)
 gridsearch_KNC.fit(train, y_train)
 print("Meilleurs parametres: ", gridsearch_KNC.best_params_ )
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))<string_transform>
score_cv(gridsearch_KNC.best_estimator_ )
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )<save_to_csv>
score_cv(XGBClassifier() )
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )<set_options>
gradient = GradientBoostingClassifier()
gradient.fit(train, y_train )
%matplotlib inline
plt.style.use('seaborn-whitegrid')
 warnings.filterwarnings('ignore' )<load_from_csv>
score_cv(GradientBoostingClassifier() )
test = pd.read_csv('.. /input/titanic/test.csv')
 train = pd.read_csv('.. /input/titanic/train.csv' )<prepare_x_and_y>
params = {
"loss":["deviance"],
"learning_rate": [0.01, 0.05, 0.1, 0.15, 0.2],
"min_samples_split": np.linspace(0.1, 0.5, 4),
"min_samples_leaf": np.linspace(0.1, 0.5, 4),
"max_depth":[3,5,8],
"max_features":["auto","log2","sqrt"],
"criterion": ["friedman_mse", "mae"],
"subsample":[0.5, 0.618, 0.8, 0.85, 0.9, 0.95, 1.0],
"n_estimators":[100]
}
gridsearch_gradient = RandomizedSearchCV(GradientBoostingClassifier() , params, n_iter = 500, cv=5)
 gridsearch_gradient.fit(train, y_train)
 print("Meilleurs parametres: ", gridsearch_gradient.best_params_ )
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train['Survived'].values
passId = test['PassengerId']
data = pd.concat(( train, test))
print("data size is: {}".format(data.shape))<count_values>
score_cv(gridsearch_gradient.best_estimator_ )
train['Survived'].value_counts()<count_missing_values>
pred = gridsearch_logistic.best_estimator_.predict(test)
 sub = pd.DataFrame()
sub['PassengerID'] = test_ID
sub['Survived'] = pred
sub.to_csv('submission.csv',index=False )