Kaggle competition: Titanic - Machine Learning from Disaster
print('Found %s word vectors.' % len(embeddings1_index))<drop_column>
def load_titanic_data(filename, titanic_path):
csv_path = os.path.join(titanic_path, filename)
 return pd.read_csv(csv_path )
del zipfile<set_options>
train_data = load_titanic_data('train.csv',".. /input")
 test_data = load_titanic_data('test.csv','.. /input' )
gc.collect()<feature_engineering>
%matplotlib inline
train_data[['Age','SibSp','Parch','Fare']].hist(figsize=(8,8))
def build_vocab(sentences,verbose=True):
vocab={}
for sentence in tqdm(sentences,disable=(not verbose)) :
for word in sentence:
try:
vocab[word] +=1
except:
vocab[word] =1
return vocab<string_transform>
train_data['Survived'].value_counts()
sentences=df_train['question_text'].progress_apply(lambda x : x.split() ).values
vocab=build_vocab(sentences)
 print({k : vocab[k] for k in list(vocab)[:5]} )<sort_values>
train_data['Pclass'].value_counts()
def check_coverage(vocab, embeddings_index):
oov={}
a={}
i,k=0,0
for word in tqdm(vocab):
try:
a[word]=embeddings_index[word]
k+= vocab[word]
except:
oov[word]=vocab[word]
i+=vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(a)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(k /(k + i)))
 sorted_x = sorted(oov.items() , key=operator.itemgetter(1)) [::-1]
return sorted_x<compute_test_metric>
train_data['Sex'].value_counts()
oov = check_coverage(vocab,embeddings1_index )<define_variables>
train_data['Embarked'].value_counts()
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }<string_transform>
class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self.attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
return X[self.attribute_names]
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text<feature_engineering>
num_pipeline = Pipeline([ 
('select_numeric', DataFrameSelector(['Age','SibSp','Parch','Fare'])) , 
('imputer', SimpleImputer(strategy='median'))
] )
df_train['question_text']=df_train['question_text'].progress_apply(lambda x: clean_contractions(x,contraction_mapping))
df_test['question_text']=df_test['question_text'].progress_apply(lambda x: clean_contractions(x,contraction_mapping))
sentences= df_train['question_text'].apply(lambda x : x.split())
 vocab = build_vocab(sentences )<compute_test_metric>
num_pipeline.fit_transform(train_data )
oov = check_coverage(vocab,embeddings1_index )<drop_column>
class MostFrequentImputer(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
self.most_frequent_ = pd.Series([X[c].value_counts().index[0] for c in X], index=X.columns)
 return self
def transform(self, X, y=None):
return X.fillna(self.most_frequent_ )
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown<define_variables>
from sklearn.preprocessing import OneHotEncoder
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }<categorify>
cat_pipeline = Pipeline([ 
('select_cat', DataFrameSelector(['Pclass','Sex','Embarked'])) , 
('imputer', MostFrequentImputer()), 
('cat_encoder', OneHotEncoder(sparse=False))
] )
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text<feature_engineering>
cat_pipeline.fit_transform(train_data )
df_train['question_text'] = df_train['question_text'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
df_test['question_text'] = df_test['question_text'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
sentences= df_train['question_text'].apply(lambda x : x.split())
 vocab = build_vocab(sentences)
<compute_test_metric>
preprocess_pipeline = FeatureUnion(transformer_list=[ 
('num_pipeline', num_pipeline), 
('cat_pipeline', cat_pipeline),
] )
oov = check_coverage(vocab,embeddings1_index )<define_variables>
y_train = train_data['Survived']
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'pokémon': 'pokemon'}<categorify>
svm_clf = SVC(gamma='auto')
 svm_clf.fit(X_train, y_train )
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x<feature_engineering>
svm_scores = cross_val_score(svm_clf, X_train, y_train, cv=10)
 svm_scores.mean()
df_train['question_text'] = df_train['question_text'].progress_apply(lambda x: correct_spelling(x, mispell_dict))
df_test['question_text'] = df_test['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))
sentences= df_train['question_text'].progress_apply(lambda x : x.split())
 sentences = [[word for word in sentence] for sentence in tqdm(sentences)]
vocab = build_vocab(sentences )<compute_test_metric>
forest_clf = RandomForestClassifier(random_state=42, n_estimators=10)
 forest_scores = cross_val_score(forest_clf, X_train, y_train, cv=10)
 forest_scores.mean()
oov = check_coverage(vocab,embeddings1_index )<set_options>
sgd_clf = SGDClassifier(max_iter=100, random_state=42, tol=0.001)
 sgd_scores = cross_val_score(sgd_clf, X_train, y_train, cv=10)
 sgd_scores.mean()
gc.collect()<define_variables>
knn_clf = KNeighborsClassifier(n_jobs=-1, weights='distance', n_neighbors=4)
 knn_scores = cross_val_score(knn_clf, X_train, y_train, cv=10)
 knn_scores.mean()
len_voc = 95000
max_len =60<import_modules>
train_data['AgeBucket'] = train_data['Age'] // 15 * 15
train_data[['AgeBucket','Survived']].groupby(['AgeBucket'] ).mean()
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences<categorify>
train_data["RelativesOnboard"] = train_data["SibSp"] + train_data["Parch"]
pd.pivot_table(train_data, index="RelativesOnboard", values="Survived", aggfunc="mean" )
t = Tokenizer(num_words=len_voc, filters='')
 t.fit_on_texts(df_train['question_text'])
 X = t.texts_to_sequences(df_train['question_text'])
 X_test=t.texts_to_sequences(df_test['question_text'])
 X = pad_sequences(X, maxlen=max_len)
 X_test = pad_sequences(X_test, maxlen=max_len)
 word_index=t.word_index<prepare_x_and_y>
age_ix, sibsp_ix, parch_ix = 1, 2, 3
class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
def __init__(self, add_attributes = True):
self.add_attributes = add_attributes
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
if self.add_attributes:
age_bucket = X[:, age_ix] // 15 * 15
relatives_onboard = X[:, sibsp_ix] + X[:, parch_ix]
return np.c_[X, age_bucket, relatives_onboard]
else:
return np.c_[X]
y = df_train['target'].values<import_modules>
num_pipeline = Pipeline([ 
('selector', DataFrameSelector(["Age", "SibSp", "Parch", "Fare"])) , 
('imputer', SimpleImputer(strategy="median")) , 
('attribs_adder', CombinedAttributesAdder()), 
('std_scaler', StandardScaler())
 ] )
from sklearn.model_selection import train_test_split<import_modules>
num_pipeline.fit_transform(train_data )
from sklearn.model_selection import train_test_split<split>
preprocess_pipeline = FeatureUnion(transformer_list=[ 
("num_pipeline", num_pipeline), 
("cat_pipeline", cat_pipeline),
] )
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=420 )<drop_column>
param_grid = [
{'n_estimators': [50, 100, 150],
'max_depth':[5, 10]}
]
forest_clf = RandomForestClassifier(random_state=42)
 grid_search = GridSearchCV(forest_clf, param_grid, cv=10, return_train_score=True)
 grid_search.fit(X_train, y_train )
del df_train<set_options>
cvres = grid_search.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(mean_score, params )
gc.collect()<categorify>
grid_search.best_params_
def make_embed_matrix(embeddings_index, word_index, len_voc):
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = word_index
embedding_matrix = np.random.normal(emb_mean, emb_std,(len_voc, embed_size))
for word, i in word_index.items() :
if i >= len_voc:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix<drop_column>
param_distribs = {
'n_estimators': randint(low=20, high=150),
'max_features': randint(low=5, high=12),
'max_depth': randint(low=5, high=12)
 }
forest_clf = RandomForestClassifier(random_state=42)
 forest_rnd_search = RandomizedSearchCV(forest_clf, param_distributions=param_distribs, n_iter=50, cv=10, return_train_score=True,
random_state=42, verbose=2, n_jobs=-1)
 forest_rnd_search.fit(X_train, y_train )
embedding = make_embed_matrix(embeddings1_index, word_index, len_voc)
 del word_index
gc.collect()<choose_model_class>
forest_rnd_search.best_params_
early = EarlyStopping(monitor='val_loss', mode="min", patience=2 )<choose_model_class>
cvres = forest_rnd_search.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(mean_score, params )
lstm = Sequential()
lstm.add(Embedding(len_voc, 300, weights=[embedding], trainable=False))
lstm.add(Bidirectional(LSTM(units = 256, return_sequences= True)))
 lstm.add(Dropout(rate = 0.2))
lstm.add(Conv1D(128, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform"))
lstm.add(GlobalMaxPooling1D())
 lstm.add(Dropout(rate = 0.2))
lstm.add(Dense(units = 1, activation = 'sigmoid'))
lstm.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'] )<define_variables>
final_model = forest_rnd_search.best_estimator_
epochs = 3
batch_size = 128<train_model>
X_test = preprocess_pipeline.transform(test_data )
hist = lstm.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(X_val,y_val),callbacks=[early] )<import_modules>
y_pred = final_model.predict(X_test)
 y_pred
<predict_on_test><EOS> 
test_data["Survived"] = y_pred
test_data[["PassengerId","Survived"]].to_csv('submission.csv', index=False )