Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
sns.set(style="darkgrid")
 warnings.filterwarnings('ignore')
 SEED = 42
TEXT = data.Field(tokenize = 'spacy', batch_first=True, include_lengths = True)
 LABEL = data.LabelField(dtype = torch.float, batch_first=True )<split>
def concat_df(train_data, test_data):
return pd.concat([train_data, test_data], sort=True ).reset_index(drop=True)
 def divide_df(all_data):
return all_data.loc[:890], all_data.loc[891:].drop(['Survived'], axis=1)
 df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_all = concat_df(df_train, df_test)
 df_train.name = 'Training Set'
df_test.name = 'Test Set'
df_all.name = 'All Set'
dfs = [df_train, df_test]
print('Number of Training Examples = {}'.format(df_train.shape[0]))
print('Number of Test Examples = {}
'.format(df_test.shape[0]))
print('Training X Shape = {}'.format(df_train.shape))
print('Training y Shape = {}
'.format(df_train['Survived'].shape[0]))
print('Test X Shape = {}'.format(df_test.shape))
print('Test y Shape = {}
'.format(df_test.shape[0]))
print(df_train.columns)
 print(df_test.columns )
class DataFrameDataset(data.Dataset):
def __init__(self, df, fields, is_test=False, **kwargs):
examples = []
for i, row in df.iterrows() :
label = row.target_relabeled if not is_test else None
text = row.text
examples.append(data.Example.fromlist([text, label], fields))
super().__init__(examples, fields, **kwargs)
 @staticmethod
def sort_key(ex):
return len(ex.text)
 @classmethod
def splits(cls, fields, train_df, val_df=None, test_df=None, **kwargs):
train_data, val_data, test_data =(None, None, None)
 data_field = fields
if train_df is not None:
train_data = cls(train_df.copy() , data_field, **kwargs)
 if val_df is not None:
val_data = cls(val_df.copy() , data_field, **kwargs)
 if test_df is not None:
test_data = cls(test_df.copy() , data_field, False, **kwargs)
 return tuple(d for d in(train_data, val_data, test_data)if d is not None )<create_dataframe>
def display_missing(df):
for col in df.columns.tolist() :
print('{} column missing values: {}'.format(col, df[col].isnull().sum()))
print('
')
 for df in dfs:
print('{}'.format(df.name))
display_missing(df )
fields = [('text',TEXT),('label',LABEL)]
train_ds, val_ds = DataFrameDataset.splits(fields, train_df=train_df, val_df=valid_df )<load_pretrained>
age_by_pclass_sex = df_all.groupby(['Sex', 'Pclass'] ).median() ['Age']
for pclass in range(1, 4):
for sex in ['female', 'male']:
print('Median age of Pclass {} {}s: {}'.format(pclass, sex, age_by_pclass_sex[sex][pclass]))
print('Median age of all passengers: {}'.format(df_all['Age'].median()))
df_all['Age'] = df_all.groupby(['Sex', 'Pclass'])['Age'].apply(lambda x: x.fillna(x.median()))
vectors = Vectors(name='.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec', cache='./')
 MAX_VOCAB_SIZE = 100000
TEXT.build_vocab(train_ds,
max_size = MAX_VOCAB_SIZE,
vectors = vectors,
unk_init = torch.Tensor.zero_)
 LABEL.build_vocab(train_ds )<count_unique_values>
df_all[df_all['Embarked'].isnull() ]
print("Size of TEXT vocabulary:",len(TEXT.vocab))
print("Size of LABEL vocabulary:",len(LABEL.vocab))
print(TEXT.vocab.freqs.most_common(10)) 
<split>
df_all['Embarked'] = df_all['Embarked'].fillna('S' )
BATCH_SIZE = 64
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 train_iterator, valid_iterator = data.BucketIterator.splits(
(train_ds, val_ds),
batch_size = BATCH_SIZE,
sort_within_batch = True,
device = device )<init_hyperparams>
df_all[df_all['Fare'].isnull() ]
num_epochs = 25
learning_rate = 0.001
INPUT_DIM = len(TEXT.vocab)
 EMBEDDING_DIM = 300
HIDDEN_DIM = 256
OUTPUT_DIM = 1
N_LAYERS = 2
BIDIRECTIONAL = True
DROPOUT = 0.2
PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]<choose_model_class>
med_fare = df_all.groupby(['Pclass', 'Parch', 'SibSp'] ).Fare.median() [3][0][0]
df_all['Fare'] = df_all['Fare'].fillna(med_fare )
class LSTM_net(nn.Module):
def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers,
bidirectional, dropout, pad_idx):
super().__init__()
self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx = pad_idx)
 self.rnn = nn.LSTM(embedding_dim,
hidden_dim,
num_layers=n_layers,
bidirectional=bidirectional,
dropout=dropout,
batch_first=True)
 self.fc1 = nn.Linear(hidden_dim * 2, hidden_dim)
 self.fc2 = nn.Linear(hidden_dim, 1)
 self.dropout = nn.Dropout(dropout)
 def forward(self, text, text_lengths):
embedded = self.embedding(text)
 packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths, batch_first=True)
 packed_output,(hidden, cell)= self.rnn(packed_embedded)
 hidden = self.dropout(torch.cat(( hidden[-2,:,:], hidden[-1,:,:]), dim = 1))
output = self.fc1(hidden)
 output = self.dropout(self.fc2(output))
return output<choose_model_class>
idx = df_all[df_all['Deck'] == 'T'].index
df_all.loc[idx, 'Deck'] = 'A'
model = LSTM_net(INPUT_DIM,
EMBEDDING_DIM,
HIDDEN_DIM,
OUTPUT_DIM,
N_LAYERS,
BIDIRECTIONAL,
DROPOUT,
PAD_IDX )<find_best_params>
df_all_decks_survived = df_all.groupby(['Deck', 'Survived'] ).count().drop(columns=['Sex', 'Age', 'SibSp', 'Parch', 'Fare',
'Embarked', 'Pclass', 'Cabin', 'PassengerId', 'Ticket'] ).rename(columns={'Name':'Count'} ).transpose()
def get_survived_dist(df):
surv_counts = {'A':{}, 'B':{}, 'C':{}, 'D':{}, 'E':{}, 'F':{}, 'G':{}, 'M':{}}
decks = df.columns.levels[0]
for deck in decks:
for survive in range(0, 2):
surv_counts[deck][survive] = df[deck][survive][0]
df_surv = pd.DataFrame(surv_counts)
 surv_percentages = {}
for col in df_surv.columns:
surv_percentages[col] = [(count / df_surv[col].sum())* 100 for count in df_surv[col]]
return surv_counts, surv_percentages
def display_surv_dist(percentages):
df_survived_percentages = pd.DataFrame(percentages ).transpose()
deck_names =('A', 'B', 'C', 'D', 'E', 'F', 'G', 'M')
 bar_count = np.arange(len(deck_names))
bar_width = 0.85
not_survived = df_survived_percentages[0]
survived = df_survived_percentages[1]
plt.figure(figsize=(20, 10))
plt.bar(bar_count, not_survived, color='
plt.bar(bar_count, survived, bottom=not_survived, color='
plt.xlabel('Deck', size=15, labelpad=20)
 plt.ylabel('Survival Percentage', size=15, labelpad=20)
 plt.xticks(bar_count, deck_names)
 plt.tick_params(axis='x', labelsize=15)
 plt.tick_params(axis='y', labelsize=15)
 plt.legend(loc='upper left', bbox_to_anchor=(1, 1), prop={'size': 15})
 plt.title('Survival Percentage in Decks', size=18, y=1.05)
 plt.show()
all_surv_count, all_surv_per = get_survived_dist(df_all_decks_survived)
 display_surv_dist(all_surv_per )
print(model)
 def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
 print(f'The model has {count_parameters(model):,} trainable parameters' )<feature_engineering>
df_all['Deck'] = df_all['Deck'].replace(['A', 'B', 'C'], 'ABC')
 df_all['Deck'] = df_all['Deck'].replace(['D', 'E'], 'DE')
 df_all['Deck'] = df_all['Deck'].replace(['F', 'G'], 'FG')
 df_all['Deck'].value_counts()
pretrained_embeddings = TEXT.vocab.vectors
model.embedding.weight.data.copy_(pretrained_embeddings)
 model.embedding.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM )<compute_test_metric>
df_all.drop(['Cabin'], inplace=True, axis=1)
 df_train, df_test = divide_df(df_all)
 dfs = [df_train, df_test]
for df in dfs:
display_missing(df )
def binary_accuracy(preds, y):

 rounded_preds = torch.round(torch.sigmoid(preds))
correct =(rounded_preds == y ).float()
acc = correct.sum() / len(correct)
 return acc<train_on_grid>
corr = df_train_corr_nd['Correlation Coefficient'] > 0.1
df_train_corr_nd[corr]
def train(model, iterator, optimizer, criterion):
epoch_loss = 0
epoch_acc = 0
model.train()
for batch in iterator:
text, text_lengths = batch.text
optimizer.zero_grad()
predictions = model(text, text_lengths ).squeeze(1)
 loss = criterion(predictions, batch.label)
 acc = binary_accuracy(predictions, batch.label)
 loss.backward()
optimizer.step()
epoch_loss += loss.item()
epoch_acc += acc.item()
return epoch_loss / len(iterator), epoch_acc / len(iterator )<compute_train_metric>
corr = df_test_corr_nd['Correlation Coefficient'] > 0.1
df_test_corr_nd[corr]
def evaluate(model, iterator, criterion):
epoch_loss = 0
epoch_acc = 0
model.eval()
with torch.no_grad() :
for batch in iterator:
text, text_lengths = batch.text
predictions = model(text, text_lengths ).squeeze(1)
 loss = criterion(predictions, batch.label)
 acc = binary_accuracy(predictions, batch.label)
 epoch_loss += loss.item()
epoch_acc += acc.item()
return epoch_loss / len(iterator), epoch_acc / len(iterator )<choose_model_class>
df_all = concat_df(df_train, df_test)
 df_all.head()
t = time.time()
best_valid_loss = float('inf')
 model.to(device)
 criterion = nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(model.parameters() , lr=learning_rate)
 for epoch in range(num_epochs):
train_loss, train_acc = train(model, train_iterator, optimizer, criterion)
 valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)
 print(f'\tTrain Loss: {train_loss:.4f} | Train Acc: {train_acc*100:.4f}%')
 print(f'\t Val.Loss: {valid_loss:.4f} | Val.Acc: {valid_acc*100:.4f}%')
 if valid_loss < best_valid_loss:
best_valid_loss = valid_loss
torch.save(model.state_dict() , 'best_model.pt')
 print(f'time:{time.time() -t:.3f}' )<predict_on_test>
df_all['Fare'] = pd.qcut(df_all['Fare'], 13 )
nlp = spacy.load('en')
 def predict(model, sentence):
tokenized = [tok.text for tok in nlp.tokenizer(sentence)]
indexed = [TEXT.vocab.stoi[t] for t in tokenized]
length = [len(indexed)]
tensor = torch.LongTensor(indexed ).to(device)
 tensor = tensor.unsqueeze(1 ).T
length_tensor = torch.LongTensor(length)
 prediction = model(tensor, length_tensor ).squeeze(1)
 rounded_preds = torch.round(torch.sigmoid(prediction))
predict_class = rounded_preds.tolist() [0]
return predict_class<predict_on_test>
df_all['Age'] = pd.qcut(df_all['Age'], 10 )
PATH = ".. /working/best_model.pt"
model.load_state_dict(torch.load(PATH))
predicts = []
for i in range(len(test_data.text)) :
predict_class = predict(model, test_data.text[i])
 predicts.append(int(predict_class))<load_from_csv>
df_all['Ticket_Frequency'] = df_all.groupby('Ticket')['Ticket'].transform('count' )
submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 submission['target'] = predicts
submission<save_to_csv>
df_all['Title'] = df_all['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
df_all['Is_Married'] = 0
df_all['Is_Married'].loc[df_all['Title'] == 'Mrs'] = 1
submission.to_csv('submission.csv',index=False )<merge>
def extract_surname(data):
families = []
for i in range(len(data)) :
name = data.iloc[i]
if '(' in name:
name_no_bracket = name.split('(')[0]
else:
name_no_bracket = name
family = name_no_bracket.split(',')[0]
title = name_no_bracket.split(',')[1].strip().split(' ')[0]
for c in string.punctuation:
family = family.replace(c, '' ).strip()
families.append(family)
 return families
df_all['Family'] = extract_surname(df_all['Name'])
 df_train = df_all.loc[:890]
df_test = df_all.loc[891:]
dfs = [df_train, df_test]
gt_df = pd.read_csv(".. /input/disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv", encoding='latin_1')
 gt_df = gt_df[['choose_one', 'text']]
gt_df['target'] =(gt_df['choose_one']=='Relevant' ).astype(int)
 gt_df['id'] = gt_df.index
merged_df = pd.merge(test_data, gt_df, on='id')
 merged_df<prepare_x_and_y>
mean_survival_rate = np.mean(df_train['Survived'])
 train_family_survival_rate = []
train_family_survival_rate_NA = []
test_family_survival_rate = []
test_family_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Family'][i] in family_rates:
train_family_survival_rate.append(family_rates[df_train['Family'][i]])
 train_family_survival_rate_NA.append(1)
 else:
train_family_survival_rate.append(mean_survival_rate)
 train_family_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Family'].iloc[i] in family_rates:
test_family_survival_rate.append(family_rates[df_test['Family'].iloc[i]])
 test_family_survival_rate_NA.append(1)
 else:
test_family_survival_rate.append(mean_survival_rate)
 test_family_survival_rate_NA.append(0)
 df_train['Family_Survival_Rate'] = train_family_survival_rate
df_train['Family_Survival_Rate_NA'] = train_family_survival_rate_NA
df_test['Family_Survival_Rate'] = test_family_survival_rate
df_test['Family_Survival_Rate_NA'] = test_family_survival_rate_NA
train_ticket_survival_rate = []
train_ticket_survival_rate_NA = []
test_ticket_survival_rate = []
test_ticket_survival_rate_NA = []
for i in range(len(df_train)) :
if df_train['Ticket'][i] in ticket_rates:
train_ticket_survival_rate.append(ticket_rates[df_train['Ticket'][i]])
 train_ticket_survival_rate_NA.append(1)
 else:
train_ticket_survival_rate.append(mean_survival_rate)
 train_ticket_survival_rate_NA.append(0)
 for i in range(len(df_test)) :
if df_test['Ticket'].iloc[i] in ticket_rates:
test_ticket_survival_rate.append(ticket_rates[df_test['Ticket'].iloc[i]])
 test_ticket_survival_rate_NA.append(1)
 else:
test_ticket_survival_rate.append(mean_survival_rate)
 test_ticket_survival_rate_NA.append(0)
 df_train['Ticket_Survival_Rate'] = train_ticket_survival_rate
df_train['Ticket_Survival_Rate_NA'] = train_ticket_survival_rate_NA
df_test['Ticket_Survival_Rate'] = test_ticket_survival_rate
df_test['Ticket_Survival_Rate_NA'] = test_ticket_survival_rate_NA
target_df = merged_df[['id', 'target']]
target_df<save_to_csv>
for df in [df_train, df_test]:
df['Survival_Rate'] =(df['Ticket_Survival_Rate'] + df['Family_Survival_Rate'])/ 2
df['Survival_Rate_NA'] =(df['Ticket_Survival_Rate_NA'] + df['Family_Survival_Rate_NA'])/ 2
target_df.to_csv('perfect_submission.csv', index=False )<compute_test_metric>
non_numeric_features = ['Embarked', 'Sex', 'Deck', 'Title', 'Family_Size_Grouped', 'Age', 'Fare']
for df in dfs:
for feature in non_numeric_features:
df[feature] = LabelEncoder().fit_transform(df[feature] )
target_df["predict"] = list(submission.target)
 print('\t\tCLASSIFICATIION METRICS
')
 print(metrics.classification_report(target_df.target, target_df.predict))<set_options>
cat_features = ['Pclass', 'Sex', 'Deck', 'Embarked', 'Title', 'Family_Size_Grouped']
encoded_features = []
for df in dfs:
for feature in cat_features:
encoded_feat = OneHotEncoder().fit_transform(df[feature].values.reshape(-1, 1)).toarray()
n = df[feature].nunique()
cols = ['{}_{}'.format(feature, n)for n in range(1, n + 1)]
encoded_df = pd.DataFrame(encoded_feat, columns=cols)
 encoded_df.index = df.index
encoded_features.append(encoded_df)
 df_train = pd.concat([df_train, *encoded_features[:6]], axis=1)
 df_test = pd.concat([df_test, *encoded_features[6:]], axis=1 )
plt.style.use('ggplot')
 warnings.filterwarnings('ignore')
<define_variables>
df_all = concat_df(df_train, df_test)
 drop_cols = ['Deck', 'Embarked', 'Family', 'Family_Size', 'Family_Size_Grouped', 'Survived',
'Name', 'Parch', 'PassengerId', 'Pclass', 'Sex', 'SibSp', 'Ticket', 'Title',
'Ticket_Survival_Rate', 'Family_Survival_Rate', 'Ticket_Survival_Rate_NA', 'Family_Survival_Rate_NA']
df_all.drop(columns=drop_cols, inplace=True)
 df_all.head()
def seed_everything(seed):
os.environ['PYTHONHASHSEED']=str(seed)
 tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 seed_everything(34 )<load_from_csv>
X_train = StandardScaler().fit_transform(df_train.drop(columns=drop_cols))
y_train = df_train['Survived'].values
X_test = StandardScaler().fit_transform(df_test.drop(columns=drop_cols))
print('X_train shape: {}'.format(X_train.shape))
print('y_train shape: {}'.format(y_train.shape))
print('X_test shape: {}'.format(X_test.shape))
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train.head()<categorify>
single_best_model = RandomForestClassifier(criterion='gini',
n_estimators=1100,
max_depth=5,
min_samples_split=4,
min_samples_leaf=5,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=1)
 leaderboard_model = RandomForestClassifier(criterion='gini',
n_estimators=1750,
max_depth=7,
min_samples_split=6,
min_samples_leaf=6,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=1 )
test_id = test['id']
columns = {'id', 'location'}
train = train.drop(columns = columns)
 test = test.drop(columns = columns)
 train['keyword'] = train['keyword'].fillna('unknown')
 test['keyword'] = test['keyword'].fillna('unknown')
 train['text'] = train['text'] + ' ' + train['keyword']
test['text'] = test['text'] + ' ' + test['keyword']
columns = {'keyword'}
train = train.drop(columns = columns)
 test = test.drop(columns = columns)
 total = train.append(test )<feature_engineering>
N = 5
oob = 0
probs = pd.DataFrame(np.zeros(( len(X_test), N * 2)) , columns=['Fold_{}_Prob_{}'.format(i, j)for i in range(1, N + 1)for j in range(2)])
 importances = pd.DataFrame(np.zeros(( X_train.shape[1], N)) , columns=['Fold_{}'.format(i)for i in range(1, N + 1)], index=df_all.columns)
 fprs, tprs, scores = [], [], []
skf = StratifiedKFold(n_splits=N, random_state=N, shuffle=True)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(X_train, y_train), 1):
print('Fold {}
'.format(fold))
leaderboard_model.fit(X_train[trn_idx], y_train[trn_idx])
 trn_fpr, trn_tpr, trn_thresholds = roc_curve(y_train[trn_idx], leaderboard_model.predict_proba(X_train[trn_idx])[:, 1])
 trn_auc_score = auc(trn_fpr, trn_tpr)
 val_fpr, val_tpr, val_thresholds = roc_curve(y_train[val_idx], leaderboard_model.predict_proba(X_train[val_idx])[:, 1])
 val_auc_score = auc(val_fpr, val_tpr)
 scores.append(( trn_auc_score, val_auc_score))
fprs.append(val_fpr)
 tprs.append(val_tpr)
 probs.loc[:, 'Fold_{}_Prob_0'.format(fold)] = leaderboard_model.predict_proba(X_test)[:, 0]
probs.loc[:, 'Fold_{}_Prob_1'.format(fold)] = leaderboard_model.predict_proba(X_test)[:, 1]
importances.iloc[:, fold - 1] = leaderboard_model.feature_importances_
oob += leaderboard_model.oob_score_ / N
print('Fold {} OOB Score: {}
'.format(fold, leaderboard_model.oob_score_))
print('Average OOB Score: {}'.format(oob))
total['unique word count'] = total['text'].apply(lambda x: len(set(x.split())))
 total['stopword count'] = total['text'].apply(lambda x: len([i for i in x.lower().split() if i in wordcloud.STOPWORDS]))
total['stopword ratio'] = total['stopword count'] / total['word count']
total['punctuation count'] = total['text'].apply(lambda x: len([i for i in str(x)if i in string.punctuation]))
train = total[:len(train)]
disaster = train['target'] == 1
fig, axes = plt.subplots(4, figsize=(20, 30))
graph1 = sns.kdeplot(train.loc[~disaster]['unique word count'], shade = True, label = 'Not Disaster', ax=axes[0])
 graph1 = sns.kdeplot(train.loc[disaster]['unique word count'], shade = True, label = 'Disaster', ax=axes[0])
 graph1.set_title('Distribution of Unique Word Count')
 graph2 = sns.kdeplot(train.loc[~disaster]['stopword count'], shade = True, label = 'Not Disaster', ax=axes[1])
 graph2 = sns.kdeplot(train.loc[disaster]['stopword count'], shade = True, label = 'Disaster', ax=axes[1])
 graph2.set_title('Distribution of Stopword Word Count')
 graph3 = sns.kdeplot(train.loc[~disaster]['punctuation count'], shade = True, label = 'Not Disaster', ax=axes[2], bw = 1)
 graph3 = sns.kdeplot(train.loc[disaster]['punctuation count'], shade = True, label = 'Disaster', ax=axes[2], bw = 1)
 graph3.set_title('Distribution of Punctuation Count')
 graph4 = sns.kdeplot(train.loc[~disaster]['stopword ratio'], shade = True, label = 'Not Disaster', ax=axes[3], bw =.05)
 graph4 = sns.kdeplot(train.loc[disaster]['stopword ratio'], shade = True, label = 'Disaster', ax=axes[3], bw =.05)
 graph4.set_title('Distribution of Stopword Ratio')
 fig.tight_layout()
plt.show()<string_transform>
class_survived = [col for col in probs.columns if col.endswith('Prob_1')]
probs['1'] = probs[class_survived].sum(axis=1)/ N
probs['0'] = probs.drop(columns=class_survived ).sum(axis=1)/ N
probs['pred'] = 0
pos = probs[probs['1'] >= 0.5].index
probs.loc[pos, 'pred'] = 1
y_pred = probs['pred'].astype(int)
 submission_df = pd.DataFrame(columns=['PassengerId', 'Survived'])
 submission_df['PassengerId'] = df_test['PassengerId']
submission_df['Survived'] = y_pred.values
submission_df.to_csv('submissions.csv', header=True, index=False)
 submission_df.head(10 )