Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df["question_text"] = train_df["question_text"].str.lower()
test_df["question_text"] = test_df["question_text"].str.lower()
def clean_text1(x):
for i in range(0, 1306121):
x=re.sub('[^a-zA-Z]', ' ', x[i])
 x=x.lower()
x=x.split()
x=' '.join(x)
 return x
def clean_text(x):
for i in range(0, 56369):
x=re.sub('[^a-zA-Z]', ' ', x[i])
 x=x.lower()
x=x.split()
x=' '.join(x)
 return x
train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_text1(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8"))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary())
 model.fit(train_X, train_y, batch_size=512, epochs=2,validation_data=(val_X, val_y))
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1)
 pred_val_y = pred_glove_val_y
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_val_y>thresh ).astype(int))))
pred_test_y = pred_glove_test_y
pred_test_y =(pred_test_y>0.35 ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False)
<define_variables>
titanic_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 titanic_df.head()
np.random.seed(481945 )<import_modules>
def get_combined_data() :
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 targets = train.Survived
train.drop('Survived', 1, inplace=True)
 combined = train.append(test)
 combined.reset_index(inplace=True)
 combined.drop('index', inplace=True, axis=1)
 return combined
from sklearn.metrics import pairwise<import_modules>
combined['Cabin'][combined.Cabin.isnull() ] = 'U0'
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from sklearn import metrics<categorify>
def get_titles() :
global combined
combined['Title'] = combined['Name'].map(lambda name:name.split(',')[1].split('.')[0].strip())
 Title_Dictionary = {
"Capt": "Officer",
"Col": "Officer",
"Major": "Officer",
"Jonkheer": "Royalty",
"Don": "Royalty",
"Sir" : "Royalty",
"Dr": "Officer",
"Rev": "Officer",
"the Countess":"Royalty",
"Dona": "Royalty",
"Mme": "Mrs",
"Mlle": "Miss",
"Ms": "Mrs",
"Mr" : "Mr",
"Mrs" : "Mrs",
"Miss" : "Miss",
"Master" : "Master",
"Lady" : "Royalty"
}
combined['Title'] = combined.Title.map(Title_Dictionary )
def summary(model, input_size, batch_size=-1, device="cuda", input_type=torch.float32):
def register_hook(module):
def hook(module, input, output):
class_name = str(module.__class__ ).split(".")[-1].split("'")[0]
module_idx = len(summary)
 m_key = "%s-%i" %(class_name, module_idx + 1)
 summary[m_key] = OrderedDict()
summary[m_key]["input_shape"] = list(input[0].size())
 summary[m_key]["input_shape"][0] = batch_size
if isinstance(output,(list, tuple)) :
summary[m_key]["output_shape"] = [
[-1] + list(o.size())[1:] for o in output
]
else:
summary[m_key]["output_shape"] = list(output.size())
 summary[m_key]["output_shape"][0] = batch_size
params = 0
if hasattr(module, "weight")and hasattr(module.weight, "size"):
params += torch.prod(torch.LongTensor(list(module.weight.size())))
 summary[m_key]["trainable"] = module.weight.requires_grad
if hasattr(module, "bias")and hasattr(module.bias, "size"):
params += torch.prod(torch.LongTensor(list(module.bias.size())))
 summary[m_key]["nb_params"] = params
if(
 not isinstance(module, nn.Sequential)
 and not isinstance(module, nn.ModuleList)
 and not(module == model)
):
hooks.append(module.register_forward_hook(hook))
device = device.lower()
assert device in [
"cuda",
"cpu",
], "Input device is not valid, please specify 'cuda' or 'cpu'"
if device == "cuda" and torch.cuda.is_available() :
dtype = torch.cuda.FloatTensor
else:
dtype = torch.FloatTensor
net_device = next(model.parameters() ).device
x = torch.zeros(( 1,)+ input_size ,dtype=input_type ).to(net_device)
 summary = OrderedDict()
hooks = []
model.apply(register_hook)
 model(x)
 for h in hooks:
h.remove()
print("----------------------------------------------------------------")
 line_new = "{:>20} {:>25} {:>15}".format("Layer(type)", "Output Shape", "Param
print(line_new)
 print("================================================================")
 total_params = 0
total_output = 0
trainable_params = 0
for layer in summary:
line_new = "{:>20} {:>25} {:>15}".format(
 layer,
str(summary[layer]["output_shape"]),
"{0:,}".format(summary[layer]["nb_params"]), 
)
 total_params += summary[layer]["nb_params"]
total_output += np.prod(summary[layer]["output_shape"])
 if "trainable" in summary[layer]:
if summary[layer]["trainable"] == True:
trainable_params += summary[layer]["nb_params"]
print(line_new)
 total_input_size = abs(np.prod(input_size)* batch_size * 4./(1024 ** 2.))
 total_output_size = abs(2.* total_output * 4./(1024 ** 2.))
 total_params_size = abs(total_params.numpy() * 4./(1024 ** 2.))
 total_size = total_params_size + total_output_size + total_input_size
print("================================================================")
 print("Total params: {0:,}".format(total_params))
print("Trainable params: {0:,}".format(trainable_params))
print("Non-trainable params: {0:,}".format(total_params - trainable_params))
print("----------------------------------------------------------------")
 print("Input size(MB): %0.2f" % total_input_size)
 print("Forward/backward pass size(MB): %0.2f" % total_output_size)
 print("Params size(MB): %0.2f" % total_params_size)
 print("Estimated Total Size(MB): %0.2f" % total_size)
 print("----------------------" )<train_model>
grouped_train = combined.head(891 ).groupby(['Sex','Pclass','Title'])
 grouped_median_train = grouped_train.median()
grouped_test = combined.iloc[891:].groupby(['Sex','Pclass','Title'])
 grouped_median_test = grouped_test.median()
grouped_median_train
class CyclicLR(_LRScheduler):

 def __init__(self,
optimizer,
base_lr=1e-3,
max_lr=6e-3,
step_size_up=2000,
step_size_down=None,
mode='triangular',
gamma=1.,
scale_fn=None,
scale_mode='cycle',
last_batch_idx=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
base_lrs = self._format_lr('base_lr', optimizer, base_lr)
 if last_batch_idx == -1:
for base_lr, group in zip(base_lrs, optimizer.param_groups):
group['lr'] = base_lr
self.max_lrs = self._format_lr('max_lr', optimizer, max_lr)
 step_size_down = step_size_down or step_size_up
self.total_size = float(step_size_up + step_size_down)
 self.step_ratio = float(step_size_up)/ self.total_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
super(CyclicLR, self ).__init__(optimizer, last_batch_idx)
 def _format_lr(self, name, optimizer, lr):

 if isinstance(lr,(list, tuple)) :
if len(lr)!= len(optimizer.param_groups):
raise ValueError("expected {} values for {}, got {}".format(
 len(optimizer.param_groups), name, len(lr)))
 return torch.tensor(lr)
 else:
return lr * torch.ones(len(optimizer.param_groups))
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):

 cycle = math.floor(1 + self.last_epoch / self.total_size)
 x = 1 + self.last_epoch / self.total_size - cycle
if x <= self.step_ratio:
scale_factor = x / self.step_ratio
else:
scale_factor =(x - 1)/(self.step_ratio - 1)
 lrs = []
for base_lr, max_lr in zip(self.base_lrs, self.max_lrs):
base_height =(max_lr - base_lr)* scale_factor
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_epoch)
 lrs.append(lr)
 return lrs<load_from_csv>
def process_age() :
global combined
def fillAges(row, grouped_median):
if row['Sex']=='female' and row['Pclass'] == 1:
if row['Title'] == 'Miss':
return grouped_median.loc['female', 1, 'Miss']['Age']
elif row['Title'] == 'Mrs':
return grouped_median.loc['female', 1, 'Mrs']['Age']
elif row['Title'] == 'Officer':
return grouped_median.loc['female', 1, 'Officer']['Age']
elif row['Title'] == 'Royalty':
return grouped_median.loc['female', 1, 'Royalty']['Age']
elif row['Sex']=='female' and row['Pclass'] == 2:
if row['Title'] == 'Miss':
return grouped_median.loc['female', 2, 'Miss']['Age']
elif row['Title'] == 'Mrs':
return grouped_median.loc['female', 2, 'Mrs']['Age']
elif row['Sex']=='female' and row['Pclass'] == 3:
if row['Title'] == 'Miss':
return grouped_median.loc['female', 3, 'Miss']['Age']
elif row['Title'] == 'Mrs':
return grouped_median.loc['female', 3, 'Mrs']['Age']
elif row['Sex']=='male' and row['Pclass'] == 1:
if row['Title'] == 'Master':
return grouped_median.loc['male', 1, 'Master']['Age']
elif row['Title'] == 'Mr':
return grouped_median.loc['male', 1, 'Mr']['Age']
elif row['Title'] == 'Officer':
return grouped_median.loc['male', 1, 'Officer']['Age']
elif row['Title'] == 'Royalty':
return grouped_median.loc['male', 1, 'Royalty']['Age']
elif row['Sex']=='male' and row['Pclass'] == 2:
if row['Title'] == 'Master':
return grouped_median.loc['male', 2, 'Master']['Age']
elif row['Title'] == 'Mr':
return grouped_median.loc['male', 2, 'Mr']['Age']
elif row['Title'] == 'Officer':
return grouped_median.loc['male', 2, 'Officer']['Age']
elif row['Sex']=='male' and row['Pclass'] == 3:
if row['Title'] == 'Master':
return grouped_median.loc['male', 3, 'Master']['Age']
elif row['Title'] == 'Mr':
return grouped_median.loc['male', 3, 'Mr']['Age']
combined.head(891 ).Age = combined.head(891 ).apply(lambda r : fillAges(r, grouped_median_train)if np.isnan(r['Age'])
 else r['Age'], axis=1)
 combined.iloc[891:].Age = combined.iloc[891:].apply(lambda r : fillAges(r, grouped_median_test)if np.isnan(r['Age'])
 else r['Age'], axis=1)
 status('age' )
quora_data = pd.read_csv('.. /input/train.csv' )<load_from_csv>
def process_names() :
global combined
combined.drop('Name',axis=1,inplace=True)
 titles_dummies = pd.get_dummies(combined['Title'],prefix='Title')
 combined = pd.concat([combined,titles_dummies],axis=1)
 combined.drop('Title',axis=1,inplace=True)
 status('names' )
quora_test_data = pd.read_csv('.. /input/test.csv' )<define_variables>
def process_embarked() :
global combined
combined.head(891 ).Embarked.fillna('S', inplace=True)
 combined.iloc[891:].Embarked.fillna('S', inplace=True)
 embarked_dummies = pd.get_dummies(combined['Embarked'],prefix='Embarked')
 combined = pd.concat([combined,embarked_dummies],axis=1)
 combined.drop('Embarked',axis=1,inplace=True)
 status('embarked' )
for s in sample[sample.target == 1].question_text:
print(s )<import_modules>
def process_cabin() :
global combined
combined.Cabin.fillna('U', inplace=True)
 combined['Cabin'] = combined['Cabin'].map(lambda c : c[0])
 cabin_dummies = pd.get_dummies(combined['Cabin'], prefix='Cabin')
 combined = pd.concat([combined,cabin_dummies], axis=1)
 combined.drop('Cabin', axis=1, inplace=True)
 status('cabin' )
from nltk.tokenize import TweetTokenizer<string_transform>
def process_sex() :
global combined
combined['Sex'] = combined['Sex'].map({'male':1,'female':0})
 status('sex' )
print(nltk.tokenize.word_tokenize("Don't spoil the movie or I'll kill you"))<string_transform>
def process_pclass() :
global combined
pclass_dummies = pd.get_dummies(combined['Pclass'], prefix="Pclass")
 combined = pd.concat([combined,pclass_dummies],axis=1)
 combined.drop('Pclass',axis=1,inplace=True)
 status('pclass' )
print(TweetTokenizer().tokenize("Don't spoil the movie or I'll kill you"))<string_transform>
combined.drop('PassengerId', inplace=True, axis=1 )
def tokenize(questions):
tokenized_questions = []
for iteration, text in enumerate(questions):
if iteration % 50000 == 0:
print(iteration, "texts tokenized")
 tokenized_questions.append([t.lower() for t in TweetTokenizer().tokenize(text)])
 return tokenized_questions<string_transform>
def process_ticket() :
global combined
def cleanTicket(ticket):
ticket = ticket.replace('.','')
 ticket = ticket.replace('/','')
 ticket = ticket.split()
ticket = map(lambda t : t.strip() , ticket)
 ticket = list(filter(lambda t : not t.isdigit() , ticket))
if len(ticket)> 0:
return ticket[0]
else:
return 'XXX'
combined['Ticket'] = combined['Ticket'].map(cleanTicket)
 tickets_dummies = pd.get_dummies(combined['Ticket'], prefix='Ticket')
 combined = pd.concat([combined, tickets_dummies], axis=1)
 combined.drop('Ticket', inplace=True, axis=1)
 status('ticket' )
dev_tokens = tokenize(quora_data.question_text )<string_transform>
def process_family() :
global combined
combined['FamilySize'] = combined['Parch'] + combined['SibSp'] + 1
combined['Singleton'] = combined['FamilySize'].map(lambda s: 1 if s == 1 else 0)
 combined['SmallFamily'] = combined['FamilySize'].map(lambda s: 1 if 2<=s<=4 else 0)
 combined['LargeFamily'] = combined['FamilySize'].map(lambda s: 1 if 5<=s else 0)
 status('family' )
test_tokens = tokenize(quora_test_data.question_text )<import_modules>
pd.isnull(combined ).sum()
import torchtext<import_modules>
def compute_score(clf, X, y, scoring='accuracy'):
xval = cross_val_score(clf, X, y, cv = 5, scoring=scoring)
 return np.mean(xval )
from collections import Counter
import itertools<split>
def recover_train_test_target() :
global combined
train0 = pd.read_csv('.. /input/train.csv')
 targets = train0.Survived
train = combined.head(891)
 test = combined.iloc[891:]
return train, test, targets
train_tokens, val_tokens, train_labels, val_labels = train_test_split(dev_tokens, quora_data.target, test_size=0.1 )<define_variables>
train.Fare.loc[50:]
word_counts = Counter(itertools.chain(*train_tokens))<count_values>
clf = RandomForestClassifier(n_estimators=50, max_features='sqrt')
 clf = clf.fit(train,targets)
 print(clf.feature_importances_ )
print(len(word_counts))
print(word_counts.most_common(10))<import_modules>
model = SelectFromModel(clf, prefit=True)
 train_reduced = model.transform(train)
 train_reduced.shape
from gensim.models.keyedvectors import KeyedVectors<load_pretrained>
run_gs = False
if run_gs:
parameter_grid = {
'max_depth' : [4, 6, 8],
'n_estimators': [50, 10],
'max_features': ['sqrt', 'auto', 'log2'],
'min_samples_split': [1, 3, 10],
'min_samples_leaf': [1, 3, 10],
'bootstrap': [True, False],
}
forest = RandomForestClassifier()
cross_validation = StratifiedKFold(targets, n_folds=5)
 grid_search = GridSearchCV(forest,
scoring='accuracy',
param_grid=parameter_grid,
cv=cross_validation)
 grid_search.fit(train, targets)
 model = grid_search
parameters = grid_search.best_params_
print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
else:
parameters = {'bootstrap': False, 'min_samples_leaf': 3, 'n_estimators': 50,
'min_samples_split': 10, 'max_features': 'sqrt', 'max_depth': 6}
model = RandomForestClassifier(**parameters)
 model.fit(train, targets )
gensim_vectors = KeyedVectors.load_word2vec_format('.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True )<feature_engineering>
compute_score(model, train, targets, scoring='accuracy' )
def filter_vectors(gensim_vectors, words):
result = {}
for w in words:
if w in gensim_vectors.vocab:
result[w] = gensim_vectors[w].copy()
return result<groupby>
output = model.predict(test ).astype(int)
 df_output = pd.DataFrame()
aux = pd.read_csv('.. /input/test.csv')
 df_output['PassengerId'] = aux['PassengerId']
df_output['Survived'] = output
df_output[['PassengerId','Survived']].to_csv('output.csv',index=False )