Kaggle competition: Titanic - Machine Learning from Disaster
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
 model = RobertaForSequenceClassification.from_pretrained('roberta-base' )<categorify>
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv' )
def prepare_features(data_set, labels=None, max_seq_length = 100,
zero_pad = True, include_special_tokens = True):
input_ids = []
attention_masks = []
for sent in data_set:
encoded_dict = tokenizer.encode_plus(
 sent,
add_special_tokens = include_special_tokens,
max_length = max_seq_length,
pad_to_max_length = zero_pad,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 if labels is not None:
labels = torch.tensor(labels)
 return input_ids, attention_masks, labels
else:
return input_ids, attention_masks<feature_engineering>
train.isnull().sum()
train_input_ids, train_attention_masks, train_labels = prepare_features(
 train['text'], train['target'])
 val_input_ids, val_attention_masks, val_labels = prepare_features(
 val['text'], val['target'])
 test_input_ids, test_attention_masks = prepare_features(
 test['text'] )<create_dataframe>
train.isnull().sum()
training_set = TensorDataset(train_input_ids, train_attention_masks, train_labels)
 validation_set = TensorDataset(val_input_ids, val_attention_masks, val_labels)
 test_set = TensorDataset(test_input_ids, test_attention_masks )<train_model>
test.isnull().sum()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = model.to(device )<set_options>
test.isnull().sum()
torch.cuda.is_available()<define_variables>
%matplotlib inline
sns.set()
BATCH_SIZE = 32
LEARNING_RATE = 1e-05
EPSILON = 1e-8
MAX_EPOCHS = 10<load_pretrained>
train_test_data = [train, test]
for dataset in train_test_data:
dataset['Title']= dataset['Name'].str.extract('([A-Za-z]+)\.', expand=False )
loading_params = {'batch_size': BATCH_SIZE,
'shuffle': True,
'drop_last': False,
'num_workers': 1}
training_loader = DataLoader(training_set, **loading_params)
 validation_loader = DataLoader(validation_set, **loading_params)
 test_loading_params = {'batch_size': BATCH_SIZE,
'shuffle': False,
'drop_last': False,
'num_workers': 1}
testing_loader = DataLoader(test_set, **test_loading_params )<choose_model_class>
train['Title'].value_counts()
loss_function = nn.CrossEntropyLoss()
optimizer = AdamW(model.parameters() ,
lr = LEARNING_RATE,
eps = EPSILON 
)
 total_steps = len(training_loader)* MAX_EPOCHS
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps = 0,
num_training_steps = total_steps )<data_type_conversions>
test['Title'].value_counts()
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))<compute_test_metric>
title_mapping = {"Mr": 0, "Miss": 1, "Mrs": 2,
"Master": 3, "Dr": 3, "Rev": 3, "Col": 3, "Major": 3, "Mlle": 3,"Countess": 3,
"Ms": 3, "Lady": 3, "Jonkheer": 3, "Don": 3, "Dona" : 3, "Mme": 3,"Capt": 3,"Sir": 3 }
for dataset in train_test_data:
dataset['Title'] = dataset['Title'].map(title_mapping )
def flat_accuracy(preds, labels):

 pred_flat = np.argmax(preds, axis=1)
 labels_flat = labels
return accuracy_score(labels_flat, pred_flat )<train_model>
train.drop('Name', axis=1, inplace=True)
 test.drop('Name', axis=1, inplace=True )
for epoch in tqdm_notebook(range(MAX_EPOCHS)) :
t0 = time.time()
total_train_loss = 0
model.train()
print("EPOCH -- {} / {}".format(epoch, MAX_EPOCHS))
for step, batch in enumerate(training_loader):
if step % 30 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {} of {}.Elapsed: {:}'.format(step, len(training_loader), elapsed))
input_ids = batch[0].to(device ).to(torch.int64)
 input_masks = batch[1].to(device ).to(torch.int64)
 labels = batch[2].to(device ).to(torch.int64)
 model.zero_grad()
loss, logits = model(input_ids,
token_type_ids=None,
attention_mask=input_masks,
labels=labels)
 logits = logits.detach().cpu().numpy()
label_ids = labels.to('cpu' ).numpy()
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_train_loss / len(training_loader)
 training_time = format_time(time.time() - t0)
 print('')
 print(' Average training loss: {0:.4f}'.format(avg_train_loss))
print(' Training epoch took: {:}'.format(training_time))
print('Running Validation')
 model.eval()
val_predictions = []
val_labels = []
for batch in validation_loader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() :
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 val_predictions.append(logits.detach().cpu().numpy())
 val_labels.append(b_labels.to('cpu' ).numpy())
 val_predictions = [item for sublist in val_predictions for item in sublist]
val_labels = [item for sublist in val_labels for item in sublist]
val_accuracy = flat_accuracy(val_predictions, val_labels)
 print(' Accuracy: {0:.4f}'.format(val_accuracy))<predict_on_test>
sex_mapping = {"male": 0, "female": 1}
for dataset in train_test_data:
dataset['Sex'] = dataset['Sex'].map(sex_mapping )
model.eval()
predictions = []
for batch in testing_loader:
batch = tuple(t.to(device)for t in batch)
 input_ids, input_masks = batch
with torch.no_grad() :
logits = model(input_ids,
token_type_ids=None,
attention_mask=input_masks)[0]
logits = logits.detach().cpu().numpy()
predictions.append(logits )<define_variables>
train["Age"].fillna(train.groupby("Title")["Age"].transform("median"), inplace=True)
 test["Age"].fillna(test.groupby("Title")["Age"].transform("median"), inplace=True )
flat_predictions = [item for sublist in predictions for item in sublist]
targets = np.argmax(flat_predictions, axis=1 ).flatten()<feature_engineering>
train.groupby("Title")["Age"].transform("median" )
test['target'] = targets<data_type_conversions>
train.isnull().sum()
tokenizer.convert_tokens_to_ids(tokenizer.tokenize('dick'))<save_to_csv>
train.isnull().sum()
submission = test[['id', 'target']].to_csv("submission_roberta.csv", index=False )<categorify>
for dataset in train_test_data:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0,
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 26), 'Age'] = 1,
dataset.loc[(dataset['Age'] > 26)&(dataset['Age'] <= 36), 'Age'] = 2,
dataset.loc[(dataset['Age'] > 36)&(dataset['Age'] <= 62), 'Age'] = 3,
dataset.loc[ dataset['Age'] > 62, 'Age'] = 4
def preprocess(texts, allowed_postags=['NOUN', "ADJ", "VERB", "ADV", "DET"]):
texts_out = []
for text in texts:
lowered_text = text.lower()
doc = nlp(lowered_text)
 tokens = [token for token in doc if not(token.is_punct |
token.is_space |
token.is_digit)]
tokens = [token for token in tokens if token.is_alpha]
lemmas = [token.lemma_ for token in tokens if token.pos_ in allowed_postags]
words = [lemma for lemma in lemmas if lemma not in stop_words]
texts_out.append(words)
 return texts_out<feature_engineering>
Pclass1 = train[train['Pclass']==1]['Embarked'].value_counts()
Pclass2 = train[train['Pclass']==2]['Embarked'].value_counts()
Pclass3 = train[train['Pclass']==3]['Embarked'].value_counts()
df = pd.DataFrame([Pclass1, Pclass2, Pclass3])
 df.index = ['1st class','2nd class', '3rd class']
df.plot(kind='bar',stacked=True, figsize=(10,5))
data_pruned = data.copy(deep=True)
 data_pruned['text'] = data_pruned['text'].progress_apply(clean_text)
 data_pruned['text'] = data_pruned['text'].progress_apply(preprocess )<feature_engineering>
for dataset in train_test_data:
dataset['Embarked']=dataset['Embarked'].fillna('S' )
test_pruned = test.copy(deep=True)
 test_pruned['text'] = test_pruned['text'].apply(clean_text)
 test_pruned['text'] = test_pruned['text'].apply(preprocess )<concatenate>
embarked_mapping = {"S": 0, "C": 1, "Q": 2}
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].map(embarked_mapping )
all_text = pd.concat([data_pruned[['text']], test_pruned[['text']]], ignore_index=True )<feature_engineering>
train["Fare"].fillna(train.groupby("Pclass")["Fare"].transform("median"), inplace=True)
 test["Fare"].fillna(test.groupby("Pclass")["Fare"].transform("median"), inplace=True)
 train.head(50 )
cv = CountVectorizer(ngram_range=(1,2))
tfidf_transformer = TfidfTransformer()
x = cv.fit_transform(all_text['text'])
 x_all_tfidf = tfidf_transformer.fit_transform(x )<train_model>
for dataset in train_test_data:
dataset.loc[ dataset['Fare'] <= 17, 'Fare'] = 0,
dataset.loc[(dataset['Fare'] > 17)&(dataset['Fare'] <= 30), 'Fare'] = 1,
dataset.loc[(dataset['Fare'] > 30)&(dataset['Fare'] <= 100), 'Fare'] = 2,
dataset.loc[ dataset['Fare'] > 100, 'Fare'] = 3
training_samples = data_pruned.shape[0]
X_train = x_all_tfidf[:training_samples,:]
X_test = x_all_tfidf[training_samples:,:]<prepare_x_and_y>
train.Cabin.value_counts()
y_train = data_pruned['target']<train_model>
for dataset in train_test_data:
dataset['Cabin']=dataset['Cabin'].str[:1]
mb_classifier = MB().fit(X_train, y_train )<compute_train_metric>
cabin_mapping = {"A": 0, "B": 0.4, "C": 0.8, "D": 1.2, "E": 1.6, "F": 2, "G": 2.4, "T": 2.8}
for dataset in train_test_data:
dataset['Cabin'] = dataset['Cabin'].map(cabin_mapping )
pred = mb_classifier.predict(X_train)
 c = classification_report(y_train,pred )<find_best_model_class>
train["Cabin"].fillna(train.groupby("Pclass")["Cabin"].transform("median"), inplace=True)
 test["Cabin"].fillna(test.groupby("Pclass")["Cabin"].transform("median"), inplace=True )
skf = StratifiedKFold(n_splits=5, random_state=RANDOM_STATE)
 total_accuracy = []
total_precision = []
total_recall = []
for train_index, val_index in skf.split(X_train, y_train):
current_X_train = X_train[train_index]
current_y_train = y_train.iloc[train_index]
current_X_val = X_train[val_index]
current_y_val = y_train.iloc[val_index]
clf = clf_svm
clf.fit(current_X_train, current_y_train)
 current_predictions = clf.predict(current_X_val)
 total_accuracy.append(accuracy_score(current_y_val, current_predictions))
total_precision.append(precision_score(current_y_val, current_predictions))
total_recall.append(recall_score(current_y_val, current_predictions))
ave_accuracy = mean(total_accuracy)
 ave_precision = mean(total_precision)
 ave_recall = mean(total_recall)
 print("Average Accuracy: {:.4f}".format(ave_accuracy))
print("Average Precision: {:.4f}".format(ave_precision))
print("Average Recall: {:.4f}".format(ave_recall)) 
<predict_on_test>
train["FamilySize"] = train["SibSp"] + train["Parch"] + 1
test["FamilySize"] = test["SibSp"] + test["Parch"] + 1
test_pruned['target'] = mb_classifier.predict(x_test )<save_to_csv>
family_mapping = {1: 0, 2: 0.4, 3: 0.8, 4: 1.2, 5: 1.6, 6: 2, 7: 2.4, 8: 2.8, 9: 3.2, 10: 3.6, 11: 4}
for dataset in train_test_data:
dataset['FamilySize'] = dataset['FamilySize'].map(family_mapping )
test_pruned[['id', 'target']].to_csv("submission_2.csv", index=False )<choose_model_class>
features_drop = ['Ticket', 'SibSp', 'Parch']
train = train.drop(features_drop, axis=1)
 test = test.drop(features_drop, axis=1)
 train = train.drop(['PassengerId'], axis=1 )
clf_svm = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto' )<predict_on_test>
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
import numpy as np
clf_svm.fit(x_train, y_train)
 pred = clf_svm.predict(x_train)
 print(classification_report(y_train,pred))<save_to_csv>
k_fold = KFold(n_splits=10, shuffle=True, random_state=0 )
test_pruned['target'] = clf_svm.predict(x_test)
 test_pruned[['id', 'target']].to_csv("submission_svm.csv", index=False )<prepare_x_and_y>
clf = KNeighborsClassifier(n_neighbors = 13)
 scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
TRAIN_VAL_SPLIT = 0.8
train = data.sample(frac=TRAIN_VAL_SPLIT, random_state=RANDOM_STATE)
 val = data[~data.index.isin(train.index)]
X_train = train['text'].values
y_train = train['target'].values
X_val = val['text'].values
y_val = val['target'].values
X_test = test['text'].values<categorify>
round(np.mean(score)*100, 2 )
def preprocess(texts, allowed_postags=['NOUN', "ADJ", "VERB", "ADV", "PROPN", "DET"]):
texts_out = []
for text in texts:
lowered_text = text.lower()
lowered_text = re.sub(r'https?://\S+|www\.\S+', '', lowered_text)
 lowered_text = re.sub(r'<.*?>', '', lowered_text)
 doc = nlp(lowered_text)
 tokens = [token for token in doc if not(token.is_punct |
token.is_space |
token.is_digit)]
lemmas = [token.lemma_ for token in tokens if token.pos_ in allowed_postags]
words = [lemma for lemma in lemmas if lemma not in stop_words]
texts_out.append(words)
 return texts_out<split>
clf = DecisionTreeClassifier()
scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
train_corpus = preprocess(X_train)
 val_corpus = preprocess(X_val)
 test_corpus = preprocess(X_test )<define_variables>
round(np.mean(score)*100, 2 )
corpus = train_corpus + val_corpus + test_corpus<string_transform>
clf = RandomForestClassifier(n_estimators=13)
 scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
tokenizer = Tokenizer()
tokenizer.fit_on_texts(corpus)
 train_sequences = tokenizer.texts_to_sequences(train_corpus)
 val_sequences = tokenizer.texts_to_sequences(val_corpus)
 test_sequences = tokenizer.texts_to_sequences(test_corpus)
 word_index = tokenizer.word_index
train_max_length = max([len(x)for x in train_sequences])
 val_max_length = max([len(x)for x in val_sequences])
 test_max_length = max([len(x)for x in test_sequences])
 max_length = max(train_max_length, val_max_length, test_max_length)
 X_train_pad = pad_sequences(train_sequences, maxlen=max_length, padding="post")
 X_val_pad = pad_sequences(val_sequences, maxlen=max_length, padding="post")
 X_test_pad = pad_sequences(test_sequences, maxlen=max_length, padding="post")
 vocab = np.array(list(tokenizer.word_index.keys()))
vocab_size = len(tokenizer.word_index)+ 1<define_variables>
round(np.mean(score)*100, 2 )
EMBEDDING_DIM = 30<choose_model_class>
clf = GaussianNB()
scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
model_tuned = Sequential()
model_tuned.add(Embedding(vocab_size, EMBEDDING_DIM, input_length=max_length))
model_tuned.add(GRU(units=30, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))
model_tuned.add(GRU(units=30, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))
model_tuned.add(GRU(units=30, dropout=0.2, recurrent_dropout=0.2,
kernel_regularizer=l1_l2(0.01, 0.01), recurrent_regularizer=l1_l2(0.01, 0.01)))
 model_tuned.add(Dense(1, activation='sigmoid'))
model_tuned.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
round(np.mean(score)*100, 2 )
history = model_tuned.fit(X_train_pad, y_train, batch_size=128, epochs=25, validation_data=(X_val_pad, y_val), verbose=2 )<predict_on_test>
clf = SVC()
scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )
y_pred_tuned = model_tuned.predict(X_test_pad )<define_variables>
round(np.mean(score)*100,2 )
y_pred_binary = list(map(lambda x: 1 if x >= 0.5 else 0, y_pred_tuned))<feature_engineering>
clf = SVC()
clf.fit(train_data, target)
 test_data = test.drop("PassengerId", axis=1 ).copy()
prediction = clf.predict(test_data )
test['target'] = y_pred_binary<save_to_csv>
submission = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": prediction
})
 submission.to_csv('submission.csv', index=False )