Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_model>
%matplotlib inline
%matplotlib inline
rcParams['figure.figsize'] = 12, 4
warnings.filterwarnings('ignore' )
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_x)+ list(test_x))<string_transform>
data_raw = pd.read_csv('.. /input/titanic/train.csv')
 data_val = pd.read_csv('.. /input/titanic/test.csv')
 data_train = data_raw.copy(deep = True)
 data_test = data_val.copy(deep = True)
 print(data_train.info())
 print("
print(data_test.info())
 print("

 data_combine = [data_train, data_test]
train_x = tokenizer.texts_to_sequences(train_x)
 test_x = tokenizer.texts_to_sequences(test_x )<concatenate>
data_train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
train_x = sequence.pad_sequences(train_x, maxlen=maxlen)
 test_x = sequence.pad_sequences(test_x, maxlen=maxlen )<compute_test_metric>
data_train[["Sex", "Survived"]].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32' )<string_transform>
data_train[["SibSp", "Survived"]].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
word_index = tokenizer.word_index<define_variables>
data_train[["Parch", "Survived"]].groupby(['Parch'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
nb_words = min(max_features, len(word_index))<define_variables>
data_train[["Embarked", "Survived"]].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
embedding_matrix = np.zeros(( nb_words, embed_size))<feature_engineering>
for dataset in data_combine:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(data_train['Title'], data_train['Sex'] )
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<compute_train_metric>
for dataset in data_combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 data_train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: %d - score: %.6f
" %(epoch+1, score))<choose_model_class>
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in data_combine:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 data_train.head()
def get_model() :
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(80, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 outp = Dense(6, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model = get_model()<define_variables>
data_train = data_train.drop(['Name', 'PassengerId'], axis=1)
 data_test = data_test.drop(['Name'], axis=1)
 data_combine = [data_train, data_test]
data_train.shape, data_test.shape
batch_size = 32
epochs = 2<split>
for dataset in data_combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 data_train.head()
X_tra, X_val, y_tra, y_val = train_test_split(train_x, train_y, train_size=0.95, random_state=233 )<compute_test_metric>
guess_ages = np.zeros(( 2,3))
guess_ages
RocAuc = RocAucEvaluation(validation_data=(X_val, y_val), interval=1 )<train_model>
for dataset in data_combine:
for i in range(0, 2):
for j in range(0, 3):
guess_df = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
guess_ages[i,j] = int(age_guess/0.5 + 0.5)* 0.5
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]
dataset['Age'] = dataset['Age'].astype(int)
 data_train.head()
hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),
callbacks=[RocAuc], verbose=2 )<predict_on_test>
data_train['AgeBand'] = pd.cut(data_train['Age'], 5)
 data_train[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False ).mean().sort_values(by='AgeBand', ascending=True )
y_pred = model.predict(test_x, batch_size=1024 )<load_from_csv>
for dataset in data_combine:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age']
data_train.head()
submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv' )<save_to_csv>
data_train = data_train.drop(['AgeBand'], axis=1)
 data_combine = [data_train, data_test]
data_train.head()
submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False )<save_to_csv>
for dataset in data_combine:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
data_train[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
submission.to_csv('submission.csv', index=False )<save_to_csv>
for dataset in data_combine:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
data_train[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean()
submission.to_csv('submission.csv', index=False )<import_modules>
data_train = data_train.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 data_test = data_test.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 data_combine = [data_train, data_test]
data_train.head()
import numpy as np
import pandas as pd
import json<load_from_csv>
for dataset in data_combine:
dataset['Age*Class'] = dataset.Age * dataset.Pclass
data_train.loc[:, ['Age*Class', 'Age', 'Pclass']].head(10 )
pd_train = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 pd_test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv' )<prepare_x_and_y>
freq_port = data_train.Embarked.dropna().mode() [0]
for dataset in data_combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
 data_train[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
train = np.array(pd_train)
 test = np.array(pd_test )<find_best_params>
for dataset in data_combine:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 data_train.head()
def find_all(input_str, search_str):
l1 = []
length = len(input_str)
 index = 0
while index < length:
i = input_str.find(search_str, index)
 if i == -1:
return l1
l1.append(i)
 index = i + 1
return l1<define_variables>
data_test['Fare'].fillna(data_test['Fare'].dropna().median() , inplace=True)
 data_test.head()
output = {}
output['version'] = 'v1.0'
output['data'] = []
for line in train:
paragraphs = []
context = line[1]
qas = []
question = line[-1]
qid = line[0]
answers = []
answer = line[2]
if type(answer)!= str or type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answer_starts = find_all(context, answer)
 for answer_start in answer_starts:
answers.append({'answer_start': answer_start, 'text': answer})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context, 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 with open('data/train.json', 'w')as outfile:
json.dump(output, outfile )<categorify>
for dataset in data_combine:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 data_train = data_train.drop(['FareBand'], axis=1)
 data_combine = [data_train, data_test]
output = {}
output['version'] = 'v1.0'
output['data'] = []
for line in test:
paragraphs = []
context = line[1]
qas = []
question = line[-1]
qid = line[0]
if type(context)!= str or type(question)!= str:
print(context, type(context))
print(answer, type(answer))
print(question, type(question))
continue
answers = []
answers.append({'answer_start': 1000000, 'text': '__None__'})
 qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})
 paragraphs.append({'context': context, 'qas': qas})
 output['data'].append({'title': 'None', 'paragraphs': paragraphs})
 with open('data/test.json', 'w')as outfile:
json.dump(output, outfile )<load_pretrained>
X_train = data_train.drop("Survived", axis=1)
 Y_train = data_train["Survived"]
X_test = data_test.drop("PassengerId", axis=1 ).copy()
X_train.shape, Y_train.shape, X_test.shape
!python /kaggle/input/pytorchtransformers/transformers-2.5.1/examples/run_squad.py \
--model_type roberta \
--model_name_or_path roberta-large \
--do_lower_case \
--do_train \
--do_eval \
--data_dir./data \
--cache_dir /kaggle/input/cached-roberta-large-pretrained/cache \
--train_file train.json \
--predict_file test.json \
--learning_rate 5e-5 \
--num_train_epochs 2 \
--max_seq_length 192 \
--doc_stride 64 \
--output_dir results_roberta_large \
--per_gpu_eval_batch_size=16 \
--per_gpu_train_batch_size=16 \
--save_steps=100000<load_from_csv>
print(X_train.columns)
 print("
print(X_test.columns)
 print("
X_train.drop("Embarked", axis=1)
 X_test.drop("Embarked", axis=1 )
predictions = json.load(open('results_roberta_large/predictions_.json', 'r'))
submission = pd.read_csv(open('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv', 'r'))
for i in range(len(submission)) :
id_ = submission['textID'][i]
if pd_test['sentiment'][i] == 'neutral':
submission.loc[i, 'selected_text'] = pd_test['text'][i]
else:
submission.loc[i, 'selected_text'] = predictions[id_]<save_to_csv>
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn import metrics
submission.to_csv('submission.csv', index=False )<import_modules>
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred = logreg.predict(X_test)
 acc_log = round(logreg.score(X_train, Y_train)* 100, 2)
 acc_log
print('Test ACC Logistic Regression -- > ', acc_log)
 X_pred = logreg.predict(X_train)
 X_predprob = logreg.predict_proba(X_train)[:,1]
t_lr_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_lr_score)
 t_lr_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_lr_roc)
 t_lr_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_lr_rmse )
print('TF version',tf.__version__ )<define_variables>
svc = SVC(probability=True)
 svc.fit(X_train, Y_train)
 Y_pred = svc.predict(X_test)
 acc_svc = round(svc.score(X_train, Y_train)* 100, 2)
 acc_svc
print('Test ACC SVM -- > ', acc_svc)
 X_pred = svc.predict(X_train)
 X_predprob = svc.predict_proba(X_train)[:,1]
t_svm_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_svm_score)
 t_svm_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_svm_roc)
 t_svm_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_svm_rmse )
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<define_variables>
param_grid = {'C': [0.1, 1, 10, 100, 1000],
'gamma': [1, 0.1, 0.01, 0.001, 0.0001],
'kernel': ['linear', 'rbf']}
grid = LinearSVC(penalty='l2', loss='squared_hinge', dual=True, tol=0.0001, C=1.0, multi_class='ovr')
 grid.fit(X_train, Y_train)
 Y_pred = grid.predict(X_test)
 acc_svc1 = round(grid.score(X_train, Y_train)* 100, 2)
 print('Test ACC LinearSVC -- > ', acc_svc )
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<feature_engineering>
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 Y_pred = knn.predict(X_test)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2)
 acc_knn
print('Test ACC KNN ', acc_knn)
 X_pred = knn.predict(X_train)
 X_predprob = knn.predict_proba(X_train)[:,1]
t_knn_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_knn_score)
 t_knn_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_knn_roc)
 t_knn_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_knn_rmse )
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('')
 ct = test.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask_t[k,:len(enc.ids)+5] = 1<load_pretrained>
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 Y_pred = gaussian.predict(X_test)
 acc_gaussian = round(gaussian.score(X_train, Y_train)* 100, 2)
 acc_gaussian
print('Test ACC Naive ', acc_gaussian)
 X_pred = gaussian.predict(X_train)
 X_predprob = gaussian.predict_proba(X_train)[:,1]
t_nb_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_nb_score)
 t_nb_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_nb_roc)
 t_nb_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_nb_rmse )
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids,attention_mask=att,token_type_ids=tok)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(1,1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(1,1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss='categorical_crossentropy', optimizer=optimizer)
 return model<string_transform>
perceptron = Perceptron()
perceptron.fit(X_train, Y_train)
 Y_pred = perceptron.predict(X_test)
 acc_perceptron = round(perceptron.score(X_train, Y_train)* 100, 2)
 acc_perceptron
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
 Y_pred = linear_svc.predict(X_test)
 acc_linear_svc = round(linear_svc.score(X_train, Y_train)* 100, 2)
 acc_linear_svc
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=5,shuffle=True,random_state=777)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model = build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5'%(VER,fold), monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch')
 model.fit([input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]], [start_tokens[idxT,], end_tokens[idxT,]],
epochs=3, batch_size=32, verbose=DISPLAY, callbacks=[sv],
validation_data=([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],
[start_tokens[idxV,], end_tokens[idxV,]]))
print('Loading model...')
 model.load_weights('%s-roberta-%i.h5'%(VER,fold))
print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<feature_engineering>
gd = GradientBoostingClassifier()
gd.fit(X_train, Y_train)
 Y_pred = gd.predict(X_test)
 acc_gd = round(gd.score(X_train, Y_train)* 100, 2)
 acc_gd
print('Test ACC Gradient Descent', acc_gd)
 X_pred = gd.predict(X_train)
 X_predprob = gd.predict_proba(X_train)[:,1]
t_gd_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_gd_score)
 t_gd_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_gd_roc)
 t_gd_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_gd_rmse )
print('>>>> OVERALL 5Fold CV Jaccard =',np.mean(jac))<string_transform>
sgd = SGDClassifier(loss='log')
 sgd.fit(X_train, Y_train)
 Y_pred = sgd.predict(X_test)
 acc_sgd = round(sgd.score(X_train, Y_train)* 100, 2)
 acc_sgd
print('Test ACC Stochastic Gradient Descent', acc_sgd)
 X_pred = sgd.predict(X_train)
 X_predprob = sgd.predict_proba(X_train)[:,1]
t_sgd_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_sgd_score)
 t_sgd_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_sgd_roc)
 t_sgd_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_sgd_rmse )
all = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-1:b])
 all.append(st )<save_to_csv>
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
 Y_pred = decision_tree.predict(X_test)
 acc_decision_tree = round(decision_tree.score(X_train, Y_train)* 100, 2)
 acc_decision_tree
print('Test ACC Decision Tree', acc_decision_tree)
 X_pred = decision_tree.predict(X_train)
 X_predprob = decision_tree.predict_proba(X_train)[:,1]
t_dt_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_dt_score)
 t_dt_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_dt_roc)
 t_dt_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_dt_rmse )
test['selected_text'] = all
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 pd.set_option('max_colwidth', 60)
 test.sample(25 )<set_options>
bag_cla = BaggingClassifier()
bag_cla.fit(X_train, Y_train)
 y_pred=bag_cla.predict(X_test)
 acc_bag_cla = round(bag_cla.score(X_train, Y_train)* 100, 2)
 print('Test ACC Bagging Classifier', acc_bag_cla)
 X_pred = bag_cla.predict(X_train)
 X_predprob = bag_cla.predict_proba(X_train)[:,1]
t_bc_score = round(metrics.accuracy_score(Y_train, X_pred)*100,2)
 print('Score -- >', t_bc_score)
 t_bc_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_bc_roc)
 t_bc_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_bc_rmse)
py.init_notebook_mode(connected=True)
 nltk.download('stopwords')
 stop=set(stopwords.words('english'))<load_from_csv>
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 Y_pred = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2)
 acc_random_forest
print('Test ACC Random Forest', acc_random_forest)
 X_pred = random_forest.predict(X_train)
 X_predprob = random_forest.predict_proba(X_train)[:,1]
t_rm_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_rm_score)
 t_rm_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_rm_roc)
 t_rm_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_rm_rmse )
def read_train() :
train=pd.read_csv(".. /input/tweet-sentiment-extraction/train.csv")
 train['text']=train['text'].astype(str)
 train['selected_text']=train['selected_text'].astype(str)
 return train
def read_test() :
test=pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 test['text']=test['text'].astype(str)
 return test
def read_submission() :
test=pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 return test
train_df = read_train()
test_df = read_test()
submission_df = read_submission() 
<string_transform>
xgb = XGBClassifier(n_estimators=100)
 xgb.fit(X_train, Y_train)
 Y_pred_xgb=xgb.predict(X_test)
 xgb.score(X_train, Y_train)
 acc_xgb = round(xgb.score(X_train, Y_train)* 100, 2)
 acc_xgb
print('Test ACC XGBoost', acc_xgb)
 X_pred = xgb.predict(X_train)
 X_predprob = xgb.predict_proba(X_train)[:,1]
t_xgb_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_xgb_score)
 t_xgb_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_xgb_roc)
 t_xgb_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_xgb_rmse)
def remove_stopwords(text):
if text is not None:
tokens = [x for x in word_tokenize(text)if x not in stop]
return " ".join(tokens)
 else:
return None
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text)
 def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text)
 def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 def remove_punct(text):
exclude = set(string.punctuation)
 s = ''.join(ch for ch in text if ch not in exclude)
 return s
def clean_df(df, train=True):
df["dirty_text"] = df['text']
df["text"] = df['text'].apply(lambda x : x.lower())
 df['text']=df['text'].apply(lambda x: remove_emoji(x))
df['text']=df['text'].apply(lambda x : remove_URL(x))
df['text']=df['text'].apply(lambda x : remove_html(x))
df['text'] =df['text'].apply(lambda x : remove_stopwords(x))
df['text']=df['text'].apply(lambda x : remove_punct(x))
df.text = df.text.replace('\s+', ' ', regex=True)
 if train:
df["selected_text"] = df['selected_text'].apply(lambda x : x.lower())
 df['selected_text']=df['selected_text'].apply(lambda x: remove_emoji(x))
df['selected_text']=df['selected_text'].apply(lambda x : remove_URL(x))
df['selected_text']=df['selected_text'].apply(lambda x : remove_html(x))
df['selected_text'] =df['selected_text'].apply(lambda x : remove_stopwords(x))
df['selected_text']=df['selected_text'].apply(lambda x : remove_punct(x))
df.selected_text = df.selected_text.replace('\s+', ' ', regex=True)
 return df 
<filter>
xgb_tuned = XGBClassifier(
 learning_rate =0.1,
n_estimators=143,
max_depth=5,
min_child_weight=1,
gamma=0.0,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27 
)
 xgb_tuned.fit(X_train, Y_train)
 Y_pred_tuned=xgb_tuned.predict(X_test)
 xgb_tuned.score(X_train, Y_train)
 acc_xgb_tuned = round(xgb_tuned.score(X_train, Y_train)* 100, 2)
 acc_xgb_tuned
print('Test ACC XGBoost Tunned', acc_xgb_tuned)
 X_pred = xgb_tuned.predict(X_train)
 X_predprob = xgb_tuned.predict_proba(X_train)[:,1]
t_xgbt_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_xgbt_score)
 t_xgbt_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_xgbt_roc)
 t_xgbt_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_xgbt_rmse )
def analyze_neutral_lenght(train_df):
neutral_df = train_df[train_df["sentiment"] == "neutral"]
equal_selected_with_text = neutral_df[neutral_df['selected_text'] == neutral_df["text"]]
print("Total number of neutral: {}".format(len(neutral_df)))
 print("Neutral with text equal to selected text: {}".format(len(equal_selected_with_text)))
 print("Ratio: {}".format(len(equal_selected_with_text)/len(neutral_df)))
 count = train_df.text.str.count(' ')<= 2
less_than_T = train_df.loc[count]
equal_selected_with_text = less_than_T[less_than_T['selected_text'] == less_than_T["text"]]
print("Total number of tweets with less than 3 words per text: {}".format(len(neutral_df)))
 print("Neutral with text equal to selected text: {}".format(len(equal_selected_with_text)))
 print("Ratio: {}".format(len(equal_selected_with_text)/len(less_than_T)))
 def post_process(submission_df, test_df):
index_to_selected_text = {}
for i, row in test_df.iterrows() :
_id = row[0]
text = row[1]
sentiment = row[2]
if len(text.split(" ")) <= 3 or sentiment == "neutral":
index_to_selected_text[_id] = text
submission_rows = submission_df.to_dict("records")
 new_rows = []
for row in submission_rows:
_id = row['textID']
if _id in index_to_selected_text:
new_row = deepcopy(row)
 new_row['selected_text'] = index_to_selected_text[_id]
else:
new_row = row
new_rows.append(new_row)
 return pd.DataFrame(new_rows)
 train_df = read_train()
test_df = read_test()
submission_df = read_submission()
analyze_neutral_lenght(train_df)
<feature_engineering>
catb=CatBoostClassifier(iterations=2500, depth=5, learning_rate=0.3, verbose=0,
allow_writing_files=False, loss_function='CrossEntropy', random_strength=0.1, leaf_estimation_method='Gradient')
 catb.fit(X_train, Y_train)
 y_pred=catb.predict(X_test)
 acc_catb = round(catb.score(X_train, Y_train)* 100, 2)
 print('Test ACC CatBoost', acc_catb)
 X_pred = catb.predict(X_train)
 X_predprob = catb.predict_proba(X_train)[:,1]
t_catb_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_catb_score)
 t_catb_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_catb_roc)
 t_catb_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_catb_rmse )
def analyze_cleaning(df, cleaning_lambda, field="selected_text"):
def diff_strings(row):
count = {}
A = row[0]
B = row[1]
if A is None:
A = "TEMP"
if B is None:
B = "TEMP"
for word in A.split() :
count[word] = count.get(word, 0)+ 1
for word in B.split() :
count[word] = count.get(word, 0)+ 1
diff = [word for word in count if count[word] == 1]
_max_len = max(len(A), len(B))
return len(diff)/_max_len
to_analyze = df[[field]]
to_analyze['cleaned_selected_text'] = to_analyze[field]
to_analyze["diff_ratio"] = None
to_analyze['cleaned_selected_text']=to_analyze['cleaned_selected_text'].apply(lambda x: cleaning_lambda(x))
to_analyze["diff_ratio"] = to_analyze.apply(lambda x: diff_strings(x), axis=1)
 print("Cleaning function used: {}".format(cleaning_lambda.__name__))
print("Average difference ratio: {}".format(to_analyze["diff_ratio"].mean()))
print("")
 train_df = read_train()
test_df = read_test()
cleaning_functions = [remove_stopwords, remove_URL, remove_html, remove_emoji, remove_punct]
for func in cleaning_functions:
analyze_cleaning(train_df, func, field="selected_text" )<string_transform>
clr = LogisticRegression()
csvc = SVC(probability=True)
 cknn = KNeighborsClassifier(n_neighbors = 3)
 cgau = GaussianNB()
cgb = GradientBoostingClassifier()
csgb = SGDClassifier(loss='log')
 crf = RandomForestClassifier(n_estimators=100)
 cxgbt = XGBClassifier(learning_rate =0.1, n_estimators=143, max_depth=5, min_child_weight=1, gamma=0.0, subsample=0.8,
colsample_bytree=0.8, objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27)
 cdt = DecisionTreeClassifier()
cbc = BaggingClassifier()
ccb = CatBoostClassifier(iterations=2500, depth=5, learning_rate=0.3, verbose=0,
allow_writing_files=False
, loss_function='CrossEntropy', random_strength=0.1, leaf_estimation_method='Gradient')

eclf1 = VotingClassifier(estimators=[('RandomForest', crf),('DecisionTree', cdt),('BaggClassifier', cbc),('CatBoost', ccb)], voting='soft')
 eclf1.fit(X_train, Y_train)
 y_pred=eclf1.predict(X_test)
 acc_eclf1 = round(eclf1.score(X_train.astype(float), Y_train.astype(float)).astype(float)* 100, 2)
 print('Test ACC Voting Classifier ', acc_eclf1)
 X_pred = eclf1.predict(X_train.astype(float))
X_predprob = eclf1.predict_proba(X_train)[:,1]
t_eclf1_score = metrics.accuracy_score(Y_train, X_pred)
 print('Score -- >', t_eclf1_score)
 t_eclf1_roc = metrics.roc_auc_score(Y_train, X_predprob)
 print('ROC -- > ', t_eclf1_roc)
 t_eclf1_rmse = metrics.mean_squared_error(Y_train, X_predprob)
 print('RMSC -- > ', t_eclf1_rmse)
 eclf1.fit(X_train, Y_train)
 y_pred=catb.predict(X_test)
 acc_eclf1 = round(eclf1.score(X_train.astype(float), Y_train.astype(float)).astype(float)* 100, 2)
<save_to_csv>

 for clf, label in zip([crf, cdt, cbc, ccb, eclf1], ['RandomForest', 'Decision Tree', 'Bagging Classifier',
'CatBoost', 'Voting Classifier']):
scores = cross_val_score(clf, X_train, Y_train, cv=5, scoring='accuracy')
 print("Accuracy: %0.2f(+/- %0.2f)[%s]" %(round(scores.mean() *100,2), scores.std() , label))
<define_variables>
models = pd.DataFrame({
'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest', 'Naive Bayes', 'Perceptron', 'Gradient Descent',
'Stochastic Gradient Descent', 'Linear SVC',
'Decision Tree', 'XgBoost', 'XgBoost_Tuned', 'Bagging Classifer', 'Cat Boost', 'Voting Classifier'],
'Score': [acc_svc, acc_knn, acc_log,
acc_random_forest, acc_gaussian, acc_perceptron,
acc_gd,acc_sgd, acc_linear_svc, acc_decision_tree, acc_xgb,acc_xgb_tuned, acc_bag_cla, acc_catb,
acc_eclf1],
'ROC': [t_svm_roc, t_knn_roc, t_lr_roc,
t_rm_roc, t_nb_roc, -100,
t_gd_roc,t_sgd_roc, -100, t_dt_roc, t_xgb_roc,t_xgbt_roc, t_bc_roc, t_catb_roc, t_eclf1_roc],
'RMSE': [t_svm_rmse, t_knn_rmse, t_lr_rmse,
t_rm_rmse, t_nb_rmse, 100,
t_gd_rmse,t_sgd_rmse, 100, t_dt_rmse, t_xgb_rmse,t_xgbt_rmse, t_bc_rmse, t_catb_rmse, t_eclf1_rmse]
})
 models.sort_values(by='RMSE', ascending=True )
<install_modules><EOS> 
X_pred = decision_tree.predict(X_train)
 submission = pd.DataFrame({
"PassengerId": data_test["PassengerId"],
"Survived": Y_pred
})
 submission.to_csv(".. /working/submission_TitanicSurvived_pred_26Feb20_1757hr.csv", index=False)
 print('Validation Data Distribution:
', submission['Survived'].value_counts(normalize = True))
submission.sample(10)
 print("Not Normalized Counts")
 submission.Survived.value_counts()