Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
%matplotlib inline
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding")
 def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
punct = "/-'?!.,
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x<drop_column>
X_full = pd.read_csv('.. /input/titanic/train.csv')
 X_test_full = pd.read_csv('.. /input/titanic/test.csv')
 X_full.dropna(axis=0, subset=['Survived'], inplace=True)
 y = X_full.Survived
X_full.drop(['PassengerId'], axis=1, inplace=True)
 X_train_full, X_valid_full, y_train, y_valid = train_test_split(X_full, y,
train_size=0.7, test_size=0.3,
random_state=2)
 fig,(train, test)= plt.subplots(ncols=2, sharey=True, figsize=(15,5))
sns.heatmap(data=X_full.isnull() , ax=train)
 sns.heatmap(data=X_test_full.isnull() , ax=test )
vocab = build_vocab(df['question_text'])
 del df<split>
categorical_cols = ['Sex', 'Embarked']
numerical_cols = ['Pclass', 'SibSp', 'Parch']
age_col = ['Age']
fare_col = ['Fare']
complex_cols = ['Name','Ticket','Cabin']
my_cols = categorical_cols + numerical_cols + age_col + fare_col + complex_cols
X_train = X_train_full[my_cols].copy()
X_valid = X_valid_full[my_cols].copy()
X_test = X_test_full[my_cols].copy()
X_train.head()
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 if file == '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if len(o)>100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index<load_from_csv>
avg_survival_chance = X_full['Survived'].mean()
avg_survival_chance_male = X_full[X_full['Sex']=='male']['Survived'].mean()
avg_survival_chance_female = X_full[X_full['Sex']=='female']['Survived'].mean()
print('all: {}, male {}, female {}'.format(avg_survival_chance, avg_survival_chance_male, avg_survival_chance_female))
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
ticket_survival = X_full[['Ticket','Survived']].copy()
ticket_survival['TicketCode'] = ticket_survival['Ticket'].map(lambda c : c.split() [0])
 ticket_survival['TicketCode'] = ticket_survival['TicketCode'].map(lambda c : 'NUM' if c.isdigit() else c)
 ticket_survival = ticket_survival.groupby('TicketCode' ).mean().sort_values(by='Survived')['Survived']
ticket_survival
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<feature_engineering>
code_dict = {}
def split_ticket_codes(surv):
if surv < 0.05:
return 1
elif surv < 0.15:
return 2
elif surv < 0.3:
return 3
elif surv < 0.4:
return 4
elif surv < 0.48:
return 5
elif surv < 0.6:
return 6
elif surv < 0.8:
return 7
return 8
for i in ticket_survival.index:
code_dict[i] = split_ticket_codes(ticket_survival[i])
 code_dict['Y'] = split_ticket_codes(avg_survival_chance )
def preprocess(df):
df["question_text"] = df["question_text"].progress_apply(lambda x: x.lower())
 df["question_text"] = df["question_text"].progress_apply(lambda x: clean_contractions(x, contraction_mapping))
df["question_text"] = df["question_text"].progress_apply(lambda x: clean_special_chars(x, punct, punct_mapping))
df["question_text"] = df["question_text"].progress_apply(lambda x: correct_spelling(x, mispell_dict))<categorify>
cabin_survival = X_full[['Cabin','Survived']].copy()
cabin_survival['Code'] = cabin_survival['Cabin'].map(lambda c : str(c)[0])
 cabin_survival = cabin_survival.groupby('Code' ).mean().sort_values(by='Survived')
 cabin_survival
preprocess(train )<split>
cabin_dict = {'T':1, 'n':2, 'A': 3, 'G':4, 'C':5, 'F':6, 'B':7, 'E':8, 'D':9}
train_df, val_df = train_test_split(train, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 100000
maxlen = 80
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<drop_column>
title_dict = {'Capt':0, 'Don':1, 'Jonkheer':2, 'Rev':3,
'Mr':4,
'Dr':7,
'Col':8, 'Major':9, 'Master':10,
'Miss':11,
'Mrs':15,
'Mme':16, 'Sir':17, 'Ms':18, 'Lady':19, 'Mlle':20, 'the Countess':21,
'Don':5, 'Jonkheer':6,
'Dona':12, 'Lady':13}
train_reserved = train
del train, train_df, val_df<load_pretrained>
train_age = X_train[['Age','Name']].append(X_test[['Age','Name']] )
glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
print("Extracting GloVe embedding")
 embed_glove = load_embed(glove )<feature_engineering>
def get_title(name):
return name.split(',')[1].split('.')[0].strip()
def get_title_id(title):
return title_dict[title] if title in title_dict else 9
def get_age_per_title(x):
x['Title'] = x.Name.map(lambda x : get_title(x))
return x.groupby('Title' ).Age.mean()
avg_age_per_title = get_age_per_title(train_age )
add_lower(embed_glove, vocab )<compute_train_metric>
train_fare = X_train[['Fare','Pclass']].append(X_test[['Fare','Pclass']] )
vocab_temp = build_vocab(train_reserved['question_text'])
 oov = check_coverage(vocab_temp, embed_glove)
 print(oov[:20])
 del oov, vocab_temp
time.sleep(5 )<feature_engineering>
def get_fare_per_pclass(x):
avg_fare = []
for i in range(1,4):
avg_fare.append(x[x['Pclass']==i]['Fare'].mean())
 print('avg_fare: {}'.format(avg_fare))
return avg_fare
avg_fare = get_fare_per_pclass(train_fare )
embeddings_index = embed_glove
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
class NameAgeFeaturesTransformer(BaseEstimator, TransformerMixin):
def __init__(self):
return
def fit(self, x, y=None):
return self
def omit_garbage(self, name):
name = name.replace('(', '(')
 name = name.replace(')', ')')
 parts = name.split()
idx_start = -1
idx_end = -1
has_nick = 0
for p in parts:
i = p.find('(')
 j = p.find(')')
 if i >= 0:
idx_start = i
if j >= 0:
idx_end = j
i = p.find('"')
 if i >= 0:
has_nick = 1
comp_len = idx_end - idx_start
return len(parts)- comp_len - has_nick
def transform(self, x):
x['Title'] = x['Name'].map(lambda name : get_title(name))
x['NameLen'] = x['Name'].map(lambda name : self.omit_garbage(name))
x['Age'] = x.apply(lambda x : avg_age_per_title[x['Title']] if pd.isnull(x['Age'])else x['Age'], axis=1)
 x['Title'] = x['Title'].map(lambda title : title_dict[title])
 x.drop('Name', axis=1, inplace=True)
 return x.values
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))<predict_on_test>
class CabinFeaturesTransformer(BaseEstimator, TransformerMixin):
def __init__(self):
return
def fit(self, x, y=None):
return self
def transform(self, x):
x.Cabin.fillna('n0', inplace=True)
 x['CabinCode'] = x['Cabin'].map(lambda c : str(c)[0])
 x['CabinCode'] = x['CabinCode'].map(lambda c : int(cabin_dict[c]))
x.drop('Cabin', axis=1, inplace=True)
 return x.values
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<drop_column>
class FareFeaturesTransformer(BaseEstimator, TransformerMixin):
def __init__(self):
return
def fit(self, x, y=None):
return self
def fillEmpty(self, fare, pclass):
if pd.isnull(fare):
return avg_fare[int(pclass)-1]
return fare
def transform(self, x):
x['Fare'] = x.apply(lambda c : self.fillEmpty(c['Fare'], c['Pclass']), axis=1)
 x.drop('Pclass', axis=1, inplace=True)
 return x.values
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x, embed_glove
time.sleep(10 )<load_pretrained>
class TicketFeaturesTransformer(BaseEstimator, TransformerMixin):
def __init__(self):
return
def fit(self, x, y=None):
return self
def transform(self, x):
x.Ticket.fillna('NUM 0', inplace=True)
 x['TicketCode'] = x['Ticket'].map(lambda c : c.split() [0])
 x['TicketCode'] = x['TicketCode'].map(lambda c : c if not(c.isdigit())else 'NUM')
 x['TicketCode'] = x['TicketCode'].map(lambda c : int(code_dict[c])if c in code_dict else code_dict['Y'])
 x['TicketNum'] = x['Ticket'].map(lambda c : c.split() [-1])
 x['TicketNum'] = x['TicketNum'].map(lambda c : int(c)if c.isdigit() else 0)
 x.drop('Ticket', axis=1, inplace=True)
 return x.values
wiki_news = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
print("Extracting FastText embedding")
 embed_fasttext = load_embed(wiki_news )<feature_engineering>
categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent')) , 
('onehot', OneHotEncoder(handle_unknown='ignore'))
])
 model = RandomForestClassifier(random_state=2)
 preprocessor = ColumnTransformer(
 transformers=[ 
('ticket', TicketFeaturesTransformer() , ['Ticket']), 
('names', NameAgeFeaturesTransformer() , ['Name','Age']), 
('cabin', CabinFeaturesTransformer() , ['Cabin']), 
('fare', FareFeaturesTransformer() , ['Fare','Pclass']), 
('cat', categorical_transformer, ['Embarked'])
 ])
 pipeline = Pipeline(steps=[('preprocessor', preprocessor)
 ,('model', model)
 ])
 pipeline.fit(X_train, y_train )
add_lower(embed_fasttext, vocab )<compute_train_metric>
parameters = {
'model__n_estimators': [560, 580, 600, 620, 640],
}
grid_search = GridSearchCV(pipeline, parameters, scoring = 'neg_mean_absolute_error', n_jobs= 1, cv=3)
 grid_search.fit(X_full, y )
vocab_temp = build_vocab(train_reserved['question_text'])
 oov = check_coverage(vocab_temp, embed_fasttext)
 print(oov[:20])
 del oov, vocab_temp
time.sleep(5 )<feature_engineering>
print(grid_search.best_params_)
 print(-1 * grid_search.best_score_ )
embeddings_index = embed_fasttext
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
model = RandomForestClassifier(random_state=2, n_estimators=580)
 pipeline = Pipeline(steps=[('preprocessor', preprocessor)
 ,('model', model)
 ])
 pipeline.fit(X_full.drop(['Survived'], axis=1), y)
 predict = pipeline.predict(X_test_full.drop(['PassengerId'], axis=1))
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y > thresh ).astype(int))))<predict_on_test>
out = pd.DataFrame({'PassengerId': X_test_full['PassengerId'], 'Survived': predict})
 out['Survived'] = out.apply(lambda x : int(x['Survived']), axis=1)
 out.to_csv('titanicOut.csv', index=False )
<drop_column><EOS> 
out = pd.read_csv('titanicOut.csv')
 out