Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<find_best_params>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn import preprocessing
from keras.models import Sequential
from keras.layers import Dense, Dropout
max_f1_index = np.argmax(f1_scores)
 best_threshold = thresholds[max_f1_index]
print(best_threshold)
 print(f1_scores[max_f1_index] )<data_type_conversions>
train_data = pd.read_csv('.. /input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv')
 train_data_copy = train_data.copy()
test_data_copy = test_data.copy()
def thresholded_predictions(probs, threshold):
return(probs > threshold ).astype(np.int8 )<compute_train_metric>
test_data['Test'] = 1
train_data['Test'] = 0
data = train_data.append(test_data, sort = False)
 drop_cols = list()
one_hot_encoding_cols = list()
normalization_cols = list()
exact_predictions = thresholded_predictions(network_pred, best_threshold)
 print(metrics.classification_report(val_labels.values, exact_predictions))<compute_test_metric>
data.drop('PassengerId', axis = 1, inplace = True )
print(metrics.confusion_matrix(val_labels.values, exact_predictions))<find_best_params>
data['Pclass'].value_counts(dropna = False)
 normalization_cols.append('Pclass' )
def test_weights(network):
network.eval()
with torch.no_grad() :
xx, yy = next(iter(val_loader))
xx, yy = xx.cuda() , yy.cuda()
print(xx[10], [vocab.itos[w] for w in xx[10]])
 weights = network.get_attention_weights(xx)
 print(weights.size())
 return weights.cpu() 
<normalization>
data['Title'] = data['Name'].str.extract('([A-Za-z]+)\.', expand=True)
 data.drop('Name', axis = 1, inplace = True )
attn = test_weights(net)
 show_attention(15,attn,val_tokens[:256] )<define_variables>
data['Title'].value_counts()
def print_examples(questions, labels, y_pred):
assert len(questions)== len(labels)== len(y_pred)
 sample = np.random.choice(len(questions),size=1000, replace=False)
 questions, labels, y_pred = questions[sample], labels[sample], y_pred[sample]
positive_pred = y_pred == 1
negative_pred = ~positive_pred
positives = labels == 1
negatives = ~positives
true_pos = questions[positive_pred & positives][:10]
false_pos = questions[positive_pred & negatives][:10]
true_neg = questions[negative_pred & negatives][:10]
false_neg = questions[negative_pred & positives][:10]
for name, examples in zip(( 'TP', 'FP', 'TN', 'FN'),(true_pos, false_pos, true_neg, false_neg)) :
print(name,":")
 for q in examples:
print("\t", q)
 print() 
<split>
mapping = {'Col': 'Army', 'Mlle' : 'Miss', 'Major' : 'Army', 'Sir': 'Royal',
'Mme': 'Mrs', 'Capt' : 'Army', 'Don' : 'Royal', 'Jonkheer' : 'Royal',
'Ms' : 'Miss', 'Countess' : 'Royal', 'Lady': 'Royal'}
data.replace({'Title': mapping}, inplace=True )
val_questions = quora_data.iloc[list(val_labels.index)].question_text.values<compute_test_metric>
data['Sex'].value_counts(dropna = False )
print_examples(val_questions, val_labels.values, exact_predictions )<find_best_params>
label_encoder = preprocessing.LabelEncoder()
data['Sex'] = label_encoder.fit_transform(data['Sex'])
 data.head()
print(net.embedding.weight.data )<data_type_conversions>
data[data['Test'] == 0].groupby(['Title', 'Sex'] ).Age.mean()
tuned_embeddings = net.embedding.weight.data.cpu().numpy()<import_modules>
data.loc[(data.Age.isna())&(data.Sex == 1)&(data.Title == 'Army'), 'Age'] = 56.60
data.loc[(data.Age.isna())&(data.Sex == 1)&(data.Title == 'Dr'), 'Age'] = 49.00
data.loc[(data.Age.isna())&(data.Sex == 0)&(data.Title == 'Dr'), 'Age'] = 40.60
data.loc[(data.Age.isna())&(data.Sex == 1)&(data.Title == 'Master'), 'Age'] = 4.57
data.loc[(data.Age.isna())&(data.Sex == 0)&(data.Title == 'Miss'), 'Age'] = 21.85
data.loc[(data.Age.isna())&(data.Sex == 1)&(data.Title == 'Mr'), 'Age'] = 32.36
data.loc[(data.Age.isna())&(data.Sex == 0)&(data.Title == 'Mrs'), 'Age'] = 35.78
data.loc[(data.Age.isna())&(data.Sex == 1)&(data.Title == 'Rev'), 'Age'] = 43.16
data.loc[(data.Age.isna())&(data.Sex == 0)&(data.Title == 'Royal'), 'Age'] = 40.50
data.loc[(data.Age.isna())&(data.Sex == 1)&(data.Title == 'Royal'), 'Age'] = 42.33
from sklearn import decomposition<normalization>
data['SibSp'].value_counts()
normalization_cols.append('SibSp' )
def get_trajectory(batch):
X = batch.cuda()
with torch.no_grad() :
states = net.get_hidden(X)
 probs = F.sigmoid(net(X)).view(-1 ).cpu().numpy()
print(states.shape)
 pca = decomposition.PCA(2)
 timesteps = states.shape[0]
batch_size = states.shape[1]
states = torch.transpose(states,0,1)
 pca_points = pca.fit_transform(states.cpu().reshape(-1,192 ).numpy())
 print(pca.explained_variance_)
 return pca_points.reshape(batch_size, timesteps, 2), probs<feature_engineering>
data['Parch'].value_counts()
normalization_cols.append('Parch')
 data.head()
batch, _ = next(iter(val_loader))
pca, probs = get_trajectory(batch)
 toks = val_tokens[:len(batch)] 
<import_modules>
data.drop('Ticket', axis =1, inplace = True )
import matplotlib.colors<define_variables>
normalization_cols.append('Fare')
cdict = {'red':(( 0.0, 0.0, 0.0), 
(1/6., 0.0, 0.0), 
(1/2., 0.8, 1.0), 
(5/6., 1.0, 1.0), 
(1.0, 0.4, 1.0)) ,
'green':(( 0.0, 0.0, 0.4), 
(1/6., 1.0, 1.0), 
(1/2., 1.0, 0.8), 
(5/6., 0.0, 0.0), 
(1.0, 0.0, 0.0)) ,
'blue':(( 0.0, 0.0, 0.0), 
(1/6., 0.0, 0.0), 
(1/2., 0.9, 0.9), 
(5/6., 0.0, 0.0), 
(1.0, 0.0, 0.0))
}
cmap=matplotlib.colors.LinearSegmentedColormap('rg',cdict, N=256 )<train_model>
data['HasCabin'] = data['Cabin'].isnull() == False
data['HasCabin'].replace(False, 0, inplace = True)
 data['HasCabin'].replace(True, 1, inplace = True)
 data.drop('Cabin', axis =1, inplace = True )
sel_word = 'iq'
nearest_neighbors(vocab, np_vectors, sel_word, 10, True)
 print('
after_training
')
 nearest_neighbors(vocab, tuned_embeddings, sel_word, 10, False )<compute_train_metric>
one_hot_encoding_cols.append('Embarked' )
def ngram_neighbors(conv_layer, vocab, vectors, idx):
ngram = conv_layer.weight.data[idx].cpu().numpy()
ngram = ngram.T
sims = pairwise.cosine_similarity(ngram, vectors)
 ranking = np.argsort(sims,axis=1)[:,::-1]
for i, row in enumerate(ranking):
print(i)
 for j in range(10):
word_index = ranking[i,j]
sim = sims[i, word_index]
print(j+1, vocab.itos[word_index], sim)
<create_dataframe>
data = pd.get_dummies(data = data, columns = one_hot_encoding_cols )
test_dataset = TokenToIdDataset(test_tokens,np.broadcast_to(np.zeros(1),shape=(len(test_tokens,))),vocab,min_size=25, precompute=True )<load_pretrained>
std = data[data['Test'] == 0][normalization_cols].std(axis = 0)
 mean = data[data['Test'] == 0][normalization_cols].mean(axis = 0)
 data[normalization_cols] =(data[normalization_cols] - mean)/ std
test_loader = torch.utils.data.DataLoader(test_dataset,batch_size=256,collate_fn=test_dataset.collate )<predict_on_test>
train_data = data[data['Test'] == 0].drop(columns = ['Test'])
 test_data = data[data['Test'] == 1].drop(columns = ['Survived', 'Test'] )
network_test_pred = predict_loader(net, test_loader )<predict_on_test>
X = train_data.iloc[: , 1:].to_numpy()
y = train_data.iloc[:, 0].to_numpy()
print(str(X.shape))
print(str(y.shape))
exact_test_predictions = thresholded_predictions(network_test_pred, best_threshold )<create_dataframe>
def create_model() :
model = Sequential()
model.add(Dense(14, input_dim = 19, activation = 'relu'))
model.add(Dropout(0.3))
model.add(Dense(8, activation = 'relu'))
model.add(Dense(1, activation = 'sigmoid'))
model.compile(loss='binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
 return model
submission = pd.DataFrame({'qid': quora_test_data.qid, 'prediction': exact_test_predictions} )<save_to_csv>
epochs = 20
model = create_model()
history = model.fit(X, y, epochs=epochs, validation_split = 0.3, batch_size=10 )
submission.to_csv('submission.csv', index=False )<save_to_csv>
epochs = 20
model = create_model()
history = model.fit(X, y, epochs=epochs, batch_size=10, verbose = 0 )
submission.to_csv('submission.csv', index=False )<save_to_csv>
X_test = test_data.to_numpy()
with open('submission.csv')as f:
for line in itertools.islice(f,0,5):
print(line )<save_to_csv>
prediction = model.predict(X_test )
exact_test_predictions = thresholded_predictions(network_test_pred, 0.5)
 submission = pd.DataFrame({'qid': quora_test_data.qid, 'prediction': exact_test_predictions})
 submission.to_csv('submission_05.csv', index=False )<import_modules>
submission = pd.DataFrame(test_data_copy[['PassengerId']])
 submission['Survived'] = prediction
submission['Survived'] = submission['Survived'].apply(lambda x: 0 if x < 0.5 else 1 )
<load_from_csv><EOS> 
submission.to_csv('submission.csv', index = False )