Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
import numpy as np
import pandas as pd
from catboost import CatBoostClassifier, Pool, cv
import hyperopt
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 submission = pd.read_csv('.. /input/sample_submission.csv' )<string_transform>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train_size = train.shape[0]
test_size = test.shape[0]
data = pd.concat([train, test] )
X_train = train["question_text"].fillna("fillna" ).values
y_train = train["target"].values
X_test = test["question_text"].fillna("fillna" ).values
max_features = 40000
maxlen = 50
embed_size = 300
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X_train)+ list(X_test))
X_train = tokenizer.texts_to_sequences(X_train)
 X_test = tokenizer.texts_to_sequences(X_test)
 x_train = sequence.pad_sequences(X_train, maxlen=maxlen)
 x_test = sequence.pad_sequences(X_test, maxlen=maxlen )<feature_engineering>
data['Title'] = data['Name'].str.extract('([A-Za-z]+)\.', expand=False )
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE))
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<predict_on_test>
age_ref = data.groupby('Title' ).Age.mean()
data['Age'] = data.apply(lambda r: r.Age if pd.notnull(r.Age)else age_ref[r.Title] , axis=1)
 del age_ref
class F1Evaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 y_pred =(y_pred > 0.35 ).astype(int)
 score = f1_score(self.y_val, y_pred)
 print("
F1 Score - epoch: %d - score: %.6f
" %(epoch+1, score))<choose_model_class>
data.loc[(data.PassengerId==1044, 'Fare')] = 14.43
filter_sizes = [1,2,3,5]
num_filters = 36
def get_model() :
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = SpatialDropout1D(0.4 )(x)
 conv_0 = Conv1D(num_filters, kernel_size=(filter_sizes[0]),
kernel_initializer='he_normal', activation='elu' )(x)
 conv_1 = Conv1D(num_filters, kernel_size=(filter_sizes[1]),
kernel_initializer='he_normal', activation='elu' )(x)
 conv_2 = Conv1D(num_filters, kernel_size=(filter_sizes[2]),
kernel_initializer='he_normal', activation='elu' )(x)
 conv_3 = Conv1D(num_filters, kernel_size=(filter_sizes[3]),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_0 = MaxPool1D(pool_size=(maxlen - filter_sizes[0] + 1))(conv_0)
 maxpool_1 = MaxPool1D(pool_size=(maxlen - filter_sizes[1] + 1))(conv_1)
 maxpool_2 = MaxPool1D(pool_size=(maxlen - filter_sizes[2] + 1))(conv_2)
 maxpool_3 = MaxPool1D(pool_size=(maxlen - filter_sizes[3] + 1))(conv_3)
 z = Concatenate(axis=1 )([maxpool_0, maxpool_1, maxpool_2, maxpool_3])
 z = Flatten()(z)
 z = BatchNormalization()(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model = get_model()<train_model>
data['Embarked'] = data['Embarked'].fillna('S')
 data['Cabin'] = data['Cabin'].fillna('Undefined' )
batch_size = 1024
epochs = 4
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.95,
random_state=233)
 F1_Score = F1Evaluation(validation_data=(X_val, y_val), interval=1)
 hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs,
validation_data=(X_val, y_val),
callbacks=[F1_Score], verbose=True )<choose_model_class>
cols = [
'Pclass',
'Name',
'Sex',
'Age',
'SibSp',
'Parch',
'Ticket',
'Fare',
'Cabin',
'Embarked'
]
X_train = data[:train_size][cols]
Y_train = data[:train_size]['Survived'].astype(int)
 X_test = data[train_size:][cols]
categorical_features_indices = [0,1,2,6,8,9]
X_train.head()
filter_sizes = [1,2,3,5]
num_filters = 36
def get_model() :
inp = Input(shape=(maxlen,))
x = Lambda(lambda x: K.reverse(x,axes=-1))(inp)
 x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(x)
 x = SpatialDropout1D(0.4 )(x)
 conv_0 = Conv1D(num_filters, kernel_size=(filter_sizes[0]),
kernel_initializer='he_normal', activation='elu' )(x)
 conv_1 = Conv1D(num_filters, kernel_size=(filter_sizes[1]),
kernel_initializer='he_normal', activation='elu' )(x)
 conv_2 = Conv1D(num_filters, kernel_size=(filter_sizes[2]),
kernel_initializer='he_normal', activation='elu' )(x)
 conv_3 = Conv1D(num_filters, kernel_size=(filter_sizes[3]),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_0 = MaxPool1D(pool_size=(maxlen - filter_sizes[0] + 1))(conv_0)
 maxpool_1 = MaxPool1D(pool_size=(maxlen - filter_sizes[1] + 1))(conv_1)
 maxpool_2 = MaxPool1D(pool_size=(maxlen - filter_sizes[2] + 1))(conv_2)
 maxpool_3 = MaxPool1D(pool_size=(maxlen - filter_sizes[3] + 1))(conv_3)
 z = Concatenate(axis=1 )([maxpool_0, maxpool_1, maxpool_2, maxpool_3])
 z = Flatten()(z)
 z = BatchNormalization()(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model_flip = get_model() 
<train_model>
train_pool = Pool(X_train, Y_train, cat_features=categorical_features_indices )
hist_flip = model_flip.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs,
validation_data=(X_val, y_val),
callbacks=[F1_Score], verbose=True )<predict_on_test>
val_y_pred1 = model.predict(X_val, batch_size=1024, verbose = True)
 val_y_pred2 = model_flip.predict(X_val, batch_size=1024, verbose = True)
<compute_test_metric>
model = CatBoostClassifier(
 depth=3,
iterations=300,
eval_metric='Accuracy',
random_seed=42,
logging_level='Silent',
allow_writing_files=False 
)
 cv_data = cv(
 train_pool,
model.get_params() ,
fold_count=5 
)
 print('Best validation accuracy score: {:.2f}±{:.2f} on step {}'.format(
 np.max(cv_data['test-Accuracy-mean']),
cv_data['test-Accuracy-std'][cv_data['test-Accuracy-mean'].idxmax(axis=0)],
cv_data['test-Accuracy-mean'].idxmax(axis=0)
))
print('Precise validation accuracy score: {}'.format(np.max(cv_data['test-Accuracy-mean'])))
 model.fit(train_pool);
model.score(X_train, Y_train )
val_y_pred = np.mean([val_y_pred1,val_y_pred2],axis = 0 )<predict_on_test>
feature_importances = model.get_feature_importance(train_pool)
 feature_names = X_train.columns
for score, name in sorted(zip(feature_importances, feature_names), reverse=True):
print('{}: {}'.format(name, score))
<compute_test_metric><EOS> 
Y_pred = model.predict(X_test)
 submission = pd.DataFrame({
"PassengerId": data[train_size:]["PassengerId"],
"Survived": Y_pred.astype(int)
 })
 submission.to_csv('submission.csv', index=False )