Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_pretrained>
warnings.filterwarnings('ignore')
 %matplotlib inline
mpl.style.use('ggplot')
 sns.set_style('white')
 pylab.rcParams[ 'figure.figsize' ] = 8 , 6
py.init_notebook_mode(connected=True)
 train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 full_data = [train_df, test_df]
train_df.head()
paragram = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
print("Extracting Paragram embedding")
 embed_paragram = load_embed(paragram )<compute_test_metric>
def check_missing_data(df):
flag=df.isna().sum().any()
if flag==True:
total = df.isnull().sum()
percent =(df.isnull().sum())/(df.isnull().count() *100)
 output = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 data_type = []
for col in df.columns:
dtype = str(df[col].dtype)
 data_type.append(dtype)
 output['Types'] = data_type
return(np.transpose(output))
else:
return(False)
 train_df = train_df.fillna(np.nan)
 test_df = test_df.fillna(np.nan)
 check_missing_data(train_df)
add_lower(embed_paragram, vocab )<compute_train_metric>
train_df[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
vocab_temp = build_vocab(train_reserved['question_text'])
 oov = check_coverage(vocab_temp, embed_paragram)
 print(oov[:20])
 del oov, vocab_temp
time.sleep(5 )<feature_engineering>
print("Before", train_df.shape, test_df.shape, full_data[0].shape, full_data[1].shape)
 train_df = train_df.drop(['Ticket', 'Cabin', 'PassengerId'], axis=1)
 test_df = test_df.drop(['Ticket', 'Cabin'], axis=1)
 full_data = [train_df, test_df]
print("After", train_df.shape, test_df.shape, full_data[0].shape, full_data[1].shape)
 for dataset in full_data:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 for dataset in full_data:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train_df[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in full_data:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 train_df = train_df.drop(['Name'], axis=1)
 test_df = test_df.drop(['Name'], axis=1)
 full_data = [train_df, test_df]
train_df.shape, test_df.shape
for dataset in full_data:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
embeddings_index = embed_paragram
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
def get_person(passenger):
age,sex = passenger
return 'child' if age < 16 else sex
train_df['Person'] = train_df[['Age','Sex']].apply(get_person,axis=1)
 test_df['Person'] = test_df[['Age','Sex']].apply(get_person,axis=1)
 person_dummies_titanic = pd.get_dummies(train_df['Person'])
 person_dummies_titanic.columns = ['Child','Female','Male']
person_dummies_titanic.drop(['Male'], axis=1, inplace=True)
 person_dummies_test = pd.get_dummies(test_df['Person'])
 person_dummies_test.columns = ['Child','Female','Male']
person_dummies_test.drop(['Male'], axis=1, inplace=True)
 train_df = train_df.join(person_dummies_titanic)
 test_df = test_df.join(person_dummies_test)
 fig,(axis1,axis2)= plt.subplots(1,2,figsize=(10,5))
sns.countplot(x='Person', data=train_df, ax=axis1)
 person_perc = train_df[["Person", "Survived"]].groupby(['Person'],as_index=False ).mean()
sns.barplot(x='Person', y='Survived', data=person_perc, ax=axis2, order=['male','female','child'])
 train_df.drop(['Person'],axis=1,inplace=True)
 test_df.drop(['Person'],axis=1,inplace=True)
 full_data = [train_df, test_df]
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))<predict_on_test>
for dataset in full_data:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
train_df[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
 for dataset in full_data:
dataset[ 'Family_Single' ] = dataset[ 'FamilySize' ].map(lambda s : 1 if s == 1 else 0)
 dataset[ 'Family_Small' ] = dataset[ 'FamilySize' ].map(lambda s : 1 if 2 <= s <= 4 else 0)
 dataset[ 'Family_Large' ] = dataset[ 'FamilySize' ].map(lambda s : 1 if 5 <= s else 0)
 train_df = train_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 test_df = test_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 full_data = [train_df, test_df]
train_df.head()
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
freq_port = train_df.Embarked.dropna().mode() [0]
for dataset in full_data:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
 train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
 for dataset in full_data:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_df.head()
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x, embed_paragram
time.sleep(10 )<compute_test_metric>
test_df['Fare'].fillna(test_df['Fare'].dropna().median() , inplace=True)
 train_df['FareBand'] = pd.qcut(train_df['Fare'], 4)
 train_df[['FareBand', 'Survived']].groupby(['FareBand'], as_index=False ).mean().sort_values(by='FareBand', ascending=True)
 for dataset in full_data:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 train_df = train_df.drop(['FareBand'], axis=1)
 full_data = [train_df, test_df]
train_df.head()
pred_val_y = 0.34*pred_glove_val_y + 0.33*pred_fasttext_val_y + 0.33*pred_paragram_val_y
best_score = 0
best_threshold = None
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_val_y>thresh))
print("F1 score at threshold {0} is {1}".format(thresh, score))
if score > best_score:
best_score = score
best_threshold = thresh
print('Best score: {0}, best threshold: {1}'.format(best_score, best_threshold))<save_to_csv>
pred_test_y = 0.34*pred_glove_test_y + 0.33*pred_fasttext_test_y + 0.33*pred_paragram_test_y
pred_test_y =(pred_test_y > best_threshold ).astype(int)
 out_df = pd.DataFrame({"qid":test["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<import_modules>
train, test = train_test_split(train_df, test_size = 0.3, random_state = 0)
 split_train_X = train.drop("Survived", axis=1)
 split_train_Y = train["Survived"]
split_test_X = test.drop("Survived", axis=1)
 split_test_Y = test["Survived"]
LSTM, GRU, GlobalMaxPool1D, Dense)
 ReduceLROnPlateau)
 tqdm(tqdm_notebook ).pandas()
print(os.listdir(".. /input"))
datapath='.. /input'
RANDOM_STATE = 2
SHUFFLE = True
TEST_SIZE = 0.8
THRESHOLD = 0.35 
<load_pretrained>
logreg = LogisticRegression()
logreg.fit(split_train_X, split_train_Y)
 logreg.score(split_test_X, split_test_Y )
@contextmanager
def timer(name):

 t0 = time.time()
yield
print(f'[{name}] done in {time.time() - t0:.0f} s')
 def load_trained_model(model, weights_path):
model.load_weights(weights_path)
 return model<load_from_csv>
gaussian = GaussianNB()
gaussian.fit(split_train_X, split_train_Y)
 gaussian.score(split_test_X, split_test_Y )
class DataReader(object):
def __init__(self,
train_file,
module,
test_file=None):
if not train_file:
raise Exception("DataReader requires a train_file!")
 if not module:
raise Exception("DataReader requires a model that can transform data!")
 self.raw_test = None
if test_file:
print("Loading test_data(%s)into dataframe" % test_file)
 self.test_data = pd.read_csv(test_file)
 self.raw_test = self.test_data[['question_text']]
print("Test data with shape: ", self.test_data.shape)
 print("Loading train_data(%s)into dataframes" % train_file)
 self.train_data = pd.read_csv(train_file)
 self.raw_train = self.train_data[['question_text']]
print("Train data with shape: ", self.train_data.shape)
 train_test_cut = self.train_data.shape[0]
if isinstance(self.raw_test, pd.DataFrame):
df_all = pd.concat([self.raw_train, self.raw_test],
axis=0 ).reset_index(drop=True)
 else:
df_all = self.raw_train
self.df_all = df_all
print("Transforming the data")
 with timer('Transforming data'):
if module:
X_features = module.transform(df_all['question_text'])
 else:
X_features = df_all['question_text']
if isinstance(X_features, list):
self.X_train = [X[:train_test_cut] for X in X_features]
self.X_test = [X[train_test_cut:] for X in X_features]
else:
self.X_train = X_features[:train_test_cut]
self.X_test = X_features[train_test_cut:]
def get_split(self, split=0.8, random_state=2, shuffle_data=True):

 print("Creating validation data by splitting(%s)" % split)
 train_data = self.train_data
X_train = self.X_train
X_t, X_v, y_t, y_v = train_test_split(
 X_train, train_data.target,
test_size=(1 - split), random_state=random_state,
shuffle=shuffle_data, stratify=train_data.target)
 return X_t, X_v, y_t, y_v
def get_kfold(self, k=5, shuffle_data=True, random_state=2):

 print("Creating validation data by kfold(%s)" % k)
 kfold = StratifiedKFold(n_splits=k, shuffle=shuffle_data, random_state=random_state)
 train_data = self.train_data
X_train = self.X_train
folded_data = kfold.split(X_train, train_data.target)
 for i in range(k):
fold = next(folded_data)
 X_t = X_train.iloc[fold[0]]
X_v = train_data.iloc[fold[0]]
y_t = X_train.iloc[fold[1]]
y_v = train_data.iloc[fold[1]]
yield X_t, X_v, y_t, y_v
def get_test(self):
if isinstance(self.test_data, pd.DataFrame):
return self.train_data, self.X_train, self.test_data, self.X_test
raise Exception("No test data provided!")
 def get_all_text(self):
return self.df_all['question_text']<feature_engineering>
perceptron = Perceptron()
perceptron.fit(split_train_X, split_train_Y)
 perceptron.score(split_test_X, split_test_Y )
class PreProcessor(object):
def __init__(self, text):
self.text = text
self.puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$',
'&', '/', '[', ']', '>', '%', '=', '
'~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<',
'→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â',
'█', '½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢',
'²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥',
'▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’',
'▀', '¨', '▄', '♫', '☆', 'é', '¯',
'♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³',
'・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√']
self.mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling',
'counselling': 'counseling',
'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary',
'Whta': 'What',
'narcisist': 'narcissist',
'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much',
'howmany': 'how many', 'whydo': 'why do',
'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
"mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum',
'narcissit': 'narcissist',
'bigdata': 'big data',
'2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend',
'airhostess': 'air hostess', "whst": 'what',
'watsapp': 'whatsapp', 'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization'}
self.mispellings_re = re.compile('(%s)' % '|'.join(self.mispell_dict.keys()))
def get_text(self):
return self.text
def replace_typical_misspell(self):
def replace(match):
return self.mispell_dict[match.group(0)]
self.text = self.mispellings_re.sub(replace, self.text)
 return self
def spacy_tokenize_words(self):
raise NotImplementedError
def normalize_unicode(self):
self.text = unicodedata.normalize('NFKD', self.text)
 return self
def remove_newline(self):

 self.text = ' '.join(self.text.split())
 return self
def decontracted(self):
text = re.sub(r"(W|w)on('|\’)t", "will not", self.text)
 text = re.sub(r"(C|c)an('|\’)t", "can not", text)
 text = re.sub(r"(Y|y )('|\’)all", "you all", text)
 text = re.sub(r"(Y|y)a('|\’)ll", "you all", text)
 text = re.sub(r"(I|i )('|\’)m", "i am", text)
 text = re.sub(r"(A|a)in('|\’)t", "aint", text)
 text = re.sub(r"n('|\’)t", " not", text)
 text = re.sub(r"('|\’)re", " are", text)
 text = re.sub(r"('|\’)s", " is", text)
 text = re.sub(r"('|\’)d", " would", text)
 text = re.sub(r"('|\’)ll", " will", text)
 text = re.sub(r"('|\’)t", " not", text)
 self.text = re.sub(r"('|\’)ve", " have", text)
 return self
def space_punctuation(self):
for punct in self.puncts:
if punct in self.text:
self.text = self.text.replace(punct, f' {punct} ')
 return self
def remove_punctuation(self):
re_tok = re.compile(f'([{string.punctuation}])')
 self.text = re_tok.sub(' ', self.text)
 return self
def clean_numbers(self):
text = self.text
if bool(re.search(r'\d', text)) :
text = re.sub('[0-9]{5,}', '
text = re.sub('[0-9]{4}', '
text = re.sub('[0-9]{3}', '
text = re.sub('[0-9]{2}', '
self.text = text
return self
def clean_and_get_text(self):
self.clean_numbers() \ 
.space_punctuation() \ 
.decontracted() \ 
.normalize_unicode() \ 
.remove_newline() \ 
.replace_typical_misspell()
return self.text<choose_model_class>
decision_tree = DecisionTreeClassifier()
decision_tree.fit(split_train_X, split_train_Y)
 decision_tree.score(split_test_X, split_test_Y )
class NeuralNetworkClassifier:

 def __init__(self, model, batch_size=512, epochs=10, val_score='val_loss',
reduce_lr=True, balancing_class_weight=False, filepath=None):

 self.model = model
self.batch_size = batch_size
self.epochs = epochs
self.val_score = val_score
self.reduce_lr = reduce_lr
self.balancing_class_weight = balancing_class_weight
self.filepath = filepath
self.model.compile(
 loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy', precision_score, recall_score,
f1_score, roc_auc_score])
 def _get_class_weight(self, y):
if self.balancing_class_weight:
return utils.class_weight.compute_class_weight(
 'balanced', np.unique(y), y)
 else:
return None
def _get_callbacks(self):
callbacks = []
callbacks.append(
 EarlyStopping(
 monitor=self.val_score,
patience=2,
verbose=1 
)
)
 if self.filepath:
callbacks.append(
 ModelCheckpoint(
 filepath=self.filepath,
monitor=self.val_score,
save_best_only=True,
save_weights_only=True 
)
)
 if self.reduce_lr:
callbacks.append(
 ReduceLROnPlateau(
 monitor=self.val_score,
factor=0.6,
patience=1,
min_lr=0.0001,
verbose=2 
)
)
 return callbacks
def predict(self, X):
return(self.predict_proba(X)> THRESHOLD ).astype(int)
 def predict_proba(self, X):
return self.model.predict([X], batch_size=1024, verbose=1)
 def train(self, X_train, y_train, X_val, y_val, verbose=1):

 callbacks = self._get_callbacks()
class_weight = self._get_class_weight(y_train)
 self.model.fit(
 X_train, y_train,
batch_size=self.batch_size,
epochs=self.epochs,
verbose=verbose,
callbacks=callbacks,
validation_data=(X_val, y_val),
shuffle=True,
class_weight=class_weight)
 return self
def fit(self, X, y, best_iteration=6, verbose=0):

 class_weight = self._get_class_weight(y)
 self.model.fit(
 X, y,
batch_size=self.batch_size,
epochs=best_iteration,
verbose=verbose,
shuffle=True,
class_weight=class_weight)
 if self.filepath:
self.model.save_weights(self.filepath)
 print('saved fitted model to {}'.format(self.filepath))
return self
@property
def best_param(self):
scores = self.model.history.history[self.val_score]
if 'loss' in self.val_score:
func = min
else:
func = max
best_iteration, _ = func(enumerate(scores), key=operator.itemgetter(1))
return best_iteration + 1
@property
def best_score(self):
scores = self.model.history.history[self.val_score]
if 'loss' in self.val_score:
func = min
else:
func = max
_, best_val_f1 = func(enumerate(scores), key=operator.itemgetter(1))
return best_val_f1

 def recall_score(y_true, y_proba, thres=THRESHOLD):

 y_pred = K.cast(K.greater(y_proba, thres), dtype='float32')
 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision_score(y_true, y_proba, thres=THRESHOLD):

 y_pred = K.cast(K.greater(y_proba, thres), dtype='float32')
 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
def f1_score(y_true, y_proba, thres=THRESHOLD):

 precision = precision_score(y_true, y_proba, thres)
 recall = recall_score(y_true, y_proba, thres)
 return 2 *(( precision * recall)/(precision + recall + K.epsilon()))
def roc_auc_score(y_true, y_proba):

 roc_auc = tf.metrics.auc(y_true, y_proba)[1]
K.get_session().run(tf.local_variables_initializer())
 return roc_auc

 class Attention(Layer):

 def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):

 def __init__(self, num_capsule, dim_capsule, routings=3,
kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule)
 class DropConnect(Wrapper):

 def __init__(self, layer, prob, **kwargs):
self.prob = prob
self.layer = layer
super(DropConnect, self ).__init__(layer, **kwargs)
 if 0.< self.prob < 1.:
self.uses_learning_phase = True
def build(self, input_shape):
if not self.layer.built:
self.layer.build(input_shape)
 self.layer.built = True
super(DropConnect, self ).build()
def compute_output_shape(self, input_shape):
return self.layer.compute_output_shape(input_shape)
 def call(self, x):
if 0.< self.prob < 1.:
self.layer.kernel = K.in_train_phase(
 K.dropout(self.layer.kernel, self.prob),
self.layer.kernel)
 self.layer.bias = K.in_train_phase(
 K.dropout(self.layer.bias, self.prob),
self.layer.bias)
 return self.layer.call(x,)
 def _dropout(x, level, noise_shape=None, seed=None):
x = K.dropout(x, level, noise_shape, seed)
 x *=(1.- level)
 return x
class QRNN(Layer):

 def __init__(self, units, window_size=2, stride=1,
return_sequences=False, go_backwards=False,
stateful=False, unroll=False, activation='tanh',
kernel_initializer='uniform', bias_initializer='zero',
kernel_regularizer=None, bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None, bias_constraint=None,
dropout=0, use_bias=True, input_dim=None, input_length=None,
**kwargs):
self.return_sequences = return_sequences
self.go_backwards = go_backwards
self.stateful = stateful
self.unroll = unroll
self.units = units
self.window_size = window_size
self.strides =(stride, 1)
 self.use_bias = use_bias
self.activation = activations.get(activation)
 self.kernel_initializer = initializers.get(kernel_initializer)
 self.bias_initializer = initializers.get(bias_initializer)
 self.kernel_regularizer = regularizers.get(kernel_regularizer)
 self.bias_regularizer = regularizers.get(bias_regularizer)
 self.activity_regularizer = regularizers.get(activity_regularizer)
 self.kernel_constraint = constraints.get(kernel_constraint)
 self.bias_constraint = constraints.get(bias_constraint)
 self.recurrent_dropout = 0
self.dropout = dropout
self.supports_masking = True
self.input_spec = [InputSpec(ndim=3)]
self.input_dim = input_dim
self.input_length = input_length
if self.input_dim:
kwargs['input_shape'] =(self.input_length, self.input_dim)
 super(QRNN, self ).__init__(**kwargs)
 def build(self, input_shape):
if isinstance(input_shape, list):
input_shape = input_shape[0]
batch_size = input_shape[0] if self.stateful else None
self.input_dim = input_shape[2]
self.input_spec = InputSpec(shape=(batch_size, None, self.input_dim))
self.state_spec = InputSpec(shape=(batch_size, self.units))
self.states = [None]
if self.stateful:
self.reset_states()
kernel_shape =(self.window_size, 1, self.input_dim, self.units * 3)
 self.kernel = self.add_weight(name='kernel',
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
 if self.use_bias:
self.bias = self.add_weight(name='bias',
shape=(self.units * 3,),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
 self.built = True
def compute_output_shape(self, input_shape):
if isinstance(input_shape, list):
input_shape = input_shape[0]
length = input_shape[1]
if length:
length = conv_output_length(length + self.window_size - 1,
self.window_size, 'valid',
self.strides[0])
 if self.return_sequences:
return(input_shape[0], length, self.units)
 else:
return(input_shape[0], self.units)
 def compute_mask(self, inputs, mask):
if self.return_sequences:
return mask
else:
return None
def get_initial_states(self, inputs):
initial_state = K.zeros_like(inputs)
 initial_state = K.sum(initial_state, axis=(1, 2))
initial_state = K.expand_dims(initial_state)
 initial_state = K.tile(initial_state, [1, self.units])
 initial_states = [initial_state for _ in range(len(self.states)) ]
return initial_states
def reset_states(self, states=None):
if not self.stateful:
raise AttributeError('Layer must be stateful.')
 if not self.input_spec:
raise RuntimeError('Layer has never been called '
'and thus has no states.')
 batch_size = self.input_spec.shape[0]
if not batch_size:
raise ValueError('If a QRNN is stateful, it needs to know '
'its batch size.Specify the batch size '
'of your input tensors:
'
'- If using a Sequential model, '
'specify the batch size by passing '
'a `batch_input_shape` '
'argument to your first layer.
 '
'- If using the functional API, specify '
'the time dimension by passing a '
'`batch_shape` argument to your Input layer.')
 if self.states[0] is None:
self.states = [K.zeros(( batch_size, self.units))
for _ in self.states]
elif states is None:
for state in self.states:
K.set_value(state, np.zeros(( batch_size, self.units)))
 else:
if not isinstance(states,(list, tuple)) :
states = [states]
if len(states)!= len(self.states):
raise ValueError('Layer ' + self.name + ' expects ' +
str(len(self.states)) + ' states, '
'but it received ' + str(len(states)) +
'state values.Input received: ' +
str(states))
for index,(value, state)in enumerate(zip(states, self.states)) :
if value.shape !=(batch_size, self.units):
raise ValueError('State ' + str(index)+
' is incompatible with layer ' +
self.name + ': expected shape=' +
str(( batch_size, self.units)) +
', found shape=' + str(value.shape))
K.set_value(state, value)
 def __call__(self, inputs, initial_state=None, **kwargs):
if initial_state is not None:
if hasattr(initial_state, '_keras_history'):
input_spec = self.input_spec
state_spec = self.state_spec
if not isinstance(state_spec, list):
state_spec = [state_spec]
self.input_spec = [input_spec] + state_spec
if not isinstance(initial_state,(list, tuple)) :
initial_state = [initial_state]
inputs = [inputs] + list(initial_state)
 output = super(QRNN, self ).__call__(inputs, **kwargs)
 self.input_spec = input_spec
return output
else:
kwargs['initial_state'] = initial_state
return super(QRNN, self ).__call__(inputs, **kwargs)
 def call(self, inputs, mask=None, initial_state=None, training=None):
if isinstance(inputs, list):
initial_states = inputs[1:]
inputs = inputs[0]
elif initial_state is not None:
pass
elif self.stateful:
initial_states = self.states
else:
initial_states = self.get_initial_states(inputs)
 if len(initial_states)!= len(self.states):
raise ValueError('Layer has ' + str(len(self.states)) +
' states but was passed ' +
str(len(initial_states)) +
' initial states.')
 input_shape = K.int_shape(inputs)
 if self.unroll and input_shape[1] is None:
raise ValueError('Cannot unroll a RNN if the '
'time dimension is undefined.
 '
'- If using a Sequential model, '
'specify the time dimension by passing '
'an `input_shape` or `batch_input_shape` '
'argument to your first layer.If your '
'first layer is an Embedding, you can '
'also use the `input_length` argument.
 '
'- If using the functional API, specify '
'the time dimension by passing a `shape` '
'or `batch_shape` argument to your Input layer.')
 constants = self.get_constants(inputs, training=None)
 preprocessed_input = self.preprocess_input(inputs, training=None)
 last_output, outputs, states = K.rnn(self.step, preprocessed_input,
initial_states,
go_backwards=self.go_backwards,
mask=mask,
constants=constants,
unroll=self.unroll,
input_length=input_shape[1])
 if self.stateful:
updates = []
for i in range(len(states)) :
updates.append(( self.states[i], states[i]))
self.add_update(updates, inputs)
 if 0 < self.dropout < 1:
last_output._uses_learning_phase = True
outputs._uses_learning_phase = True
if self.return_sequences:
return outputs
else:
return last_output
def preprocess_input(self, inputs, training=None):
if self.window_size > 1:
inputs = K.temporal_padding(inputs,(self.window_size - 1, 0))
inputs = K.expand_dims(inputs, 2)
 output = K.conv2d(inputs, self.kernel, strides=self.strides,
padding='valid',
data_format='channels_last')
 output = K.squeeze(output, 2)
 if self.use_bias:
output = K.bias_add(output, self.bias, data_format='channels_last')
 if self.dropout is not None and 0.< self.dropout < 1.:
z = output[:, :, :self.units]
f = output[:, :, self.units:2 * self.units]
o = output[:, :, 2 * self.units:]
f = K.in_train_phase(1 - _dropout(1 - f, self.dropout), f, training=training)
 return K.concatenate([z, f, o], -1)
 else:
return output
def step(self, inputs, states):
prev_output = states[0]
z = inputs[:, :self.units]
f = inputs[:, self.units:2 * self.units]
o = inputs[:, 2 * self.units:]
z = self.activation(z)
 f = f if self.dropout is not None and 0.< self.dropout < 1.else K.sigmoid(f)
 o = K.sigmoid(o)
 output = f * prev_output +(1 - f)* z
output = o * output
return output, [output]
def get_constants(self, inputs, training=None):
return []
def get_config(self):
config = {'units': self.units,
'window_size': self.window_size,
'stride': self.strides[0],
'return_sequences': self.return_sequences,
'go_backwards': self.go_backwards,
'stateful': self.stateful,
'unroll': self.unroll,
'use_bias': self.use_bias,
'dropout': self.dropout,
'activation': activations.serialize(self.activation),
'kernel_initializer': initializers.serialize(self.kernel_initializer),
'bias_initializer': initializers.serialize(self.bias_initializer),
'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),
'bias_regularizer': regularizers.serialize(self.bias_regularizer),
'activity_regularizer': regularizers.serialize(self.activity_regularizer),
'kernel_constraint': constraints.serialize(self.kernel_constraint),
'bias_constraint': constraints.serialize(self.bias_constraint),
'input_dim': self.input_dim,
'input_length': self.input_length}
base_config = super(QRNN, self ).get_config()
return dict(list(base_config.items())+ list(config.items())) 
<categorify>
random_forest = RandomForestClassifier(n_estimators=100)
 parameters = {'n_estimators': [4, 6, 9],
'max_features': ['log2', 'sqrt','auto'],
'criterion': ['entropy', 'gini'],
'max_depth': [2, 3, 5, 10],
'min_samples_split': [2, 3, 5],
'min_samples_leaf': [1,5,8]
}
acc_scorer = make_scorer(accuracy_score)
 kfold = StratifiedKFold(n_splits=10)
 grid_obj = GridSearchCV(random_forest, parameters, cv=kfold, scoring=acc_scorer)
 grid_obj.fit(X, Y)
 random_forest = grid_obj.best_estimator_
grid_obj.best_score_
def load_word_embedding(filepath):

 def _get_vec(word, *arr):
return word, np.asarray(arr, dtype='float32')
 print('Load word embeddings...')
 try:
word_embedding = dict(_get_vec(*w.split(' ')) for w in open(filepath))
except UnicodeDecodeError:
word_embedding = dict(_get_vec(*w.split(' ')) for w in open(
 filepath, encoding="utf8", errors='ignore'))
words_to_del = []
for word, vec in word_embedding.items() :
if len(vec)!= 300:
words_to_del.append(word)
 for word in words_to_del:
del word_embedding[word]
return word_embedding
def create_embedding_weights(word_index, word_embedding, max_features):

 print('Create word embeddings weights...')
 mat_embedding = np.stack(word_embedding.values())
 a, b = min(max_features, len(word_index)) , mat_embedding.shape[1]
print('Embeddings weights matrix with shape:({}, {})'.format(a, b))
embedding_mean, embedding_std = mat_embedding.mean() , mat_embedding.std()
embedding_weights = np.random.normal(embedding_mean, embedding_std,(a, b))
for word, idx in word_index.items() :
if idx >= a:
continue
word_vec = word_embedding.get(word, None)
 if word_vec is not None:
embedding_weights[idx] = word_vec
return embedding_weights<define_variables>
train_df_X = train_df.drop("Survived", axis=1)
 train_df_Y = train_df["Survived"]
test_df_X = test_df.drop("PassengerId", axis=1 ).copy()
train_df_X.shape, train_df_Y.shape, test_df_X.shape
train_X = train_df_X.values
train_Y = train_df_Y.values
test_X = test_df_X.values
classifiers = [
KNeighborsClassifier(3),
SVC(probability=True),
DecisionTreeClassifier() ,
RandomForestClassifier() ,
AdaBoostClassifier() ,
GradientBoostingClassifier() ,
GaussianNB() ,
LinearDiscriminantAnalysis() ,
QuadraticDiscriminantAnalysis() ,
LogisticRegression() ]
kfold = StratifiedKFold(n_splits=10, random_state=0)
 cv_results = []
for classifier in classifiers :
cv_results.append(cross_val_score(classifier, train_X, y = train_Y, scoring = "accuracy", cv = kfold, n_jobs=4))
cv_means = []
cv_std = []
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,"Algorithm":["SVC","DecisionTree","AdaBoost",
"RandomForest","ExtraTrees","GradientBoosting","MultipleLayerPerceptron","KNeighboors","LogisticRegression","LinearDiscriminantAnalysis"]})
 g = sns.barplot("CrossValMeans","Algorithm",data = cv_res, palette="Set3",orient = "h",**{'xerr':cv_std})
 g.set_xlabel("Mean Accuracy")
 g = g.set_title("Cross validation scores")
MAX_FEATURES = int(2.5e5)
 MAX_LEN = 80
RNN_UNITS = 40
DENSE_UNITS_1 = 32
DENSE_UNITS_2 = 16
EMBED_FILEPATH = os.path.join(datapath, 'embeddings/glove.840B.300d/glove.840B.300d.txt')
 EMBED_PICKLEPATH = 'glove.pkl'
MODEL_FILEPATH = 'submission.csv'<load_pretrained>
ensemble_lin_rbf = VotingClassifier(estimators=[('KNN', KNeighborsClassifier(n_neighbors=10)) , 
('RBF', SVC(probability=True,kernel='rbf',C=0.5,gamma=0.1)) , 
('RFor', RandomForestClassifier(n_estimators=500,random_state=0)) , 
('LR', LogisticRegression(C=0.05)) , 
('DT', DecisionTreeClassifier(random_state=0)) , 
('NB', GaussianNB()), 
('svm', SVC(kernel='linear',probability=True))
],
voting='soft' ).fit(split_train_X, split_train_Y)
 print('The accuracy for ensembled model is:', ensemble_lin_rbf.score(split_test_X, split_test_Y))
cross = cross_val_score(ensemble_lin_rbf, X, Y, cv = 10, scoring = "accuracy")
 print('The cross validated score is', cross.mean())
 model = BaggingClassifier(base_estimator = DecisionTreeClassifier() , random_state = 0, n_estimators = 100)
 model.fit(split_train_X, split_train_Y)
 prediction = model.predict(split_test_X)
 print('The accuracy for bagged Decision Tree is:', accuracy_score(prediction, split_test_Y))
result = cross_val_score(model, X, Y, cv=10, scoring='accuracy')
 print('The cross validated score for bagged Decision Tree is:', result.mean())
 ada = AdaBoostClassifier(n_estimators = 200, random_state = 0, learning_rate = 0.1)
 result = cross_val_score(ada,X, Y, cv = 10, scoring='accuracy')
 print('The cross validated score for AdaBoost is:', result.mean())
 ada = AdaBoostClassifier(n_estimators = 200, random_state = 0, learning_rate = 0.05)
 result = cross_val_predict(ada, X, Y, cv = 10)
 sns.heatmap(confusion_matrix(Y,result), cmap='winter', annot=True, fmt='2.0f')
 plt.show()
def get_network(embed_filepath):
input_layer = Input(shape=(MAX_LEN,), name='input')
 print('load pre-trained embeddings weights...')
 embed_weights = pd.read_pickle(EMBED_PICKLEPATH)
 input_dim = embed_weights.shape[0]
output_dim = embed_weights.shape[1]
x = Embedding(
 input_dim=input_dim,
output_dim=output_dim,
weights=[embed_weights],
trainable=False,
name='embeddings'
)(input_layer)
 del embed_weights, input_dim, output_dim
gc.collect()
x = SpatialDropout1D(rate=0.15 )(x)
 x = Bidirectional(
 layer=LSTM(RNN_UNITS, return_sequences=True),
name='bidirectional_lstm'
)(x)
 x = GlobalMaxPool1D(name='global_max_pooling1d' )(x)
 x = Dense(units=DENSE_UNITS_1, activation='relu', name='dense_1' )(x)
 x = Dense(units=DENSE_UNITS_2, activation='relu', name='dense_2' )(x)
 output_layer = Dense(units=1, activation='sigmoid', name='output' )(x)
 return Model(inputs=input_layer, outputs=output_layer)
 def get_model() :
print('build network...')
 model = get_network(embed_filepath=EMBED_FILEPATH)
 print(model.summary())
 return NeuralNetworkClassifier(
 model,
balancing_class_weight=True,
filepath=MODEL_FILEPATH)
 def tokenize(df_text):
def _clean(text):
return PreProcessor(text ).clean_and_get_text()
df_text = df_text.progress_apply(_clean)
 tokenizer = Tokenizer(
 num_words=MAX_FEATURES,
filters='',
lower=False,
split=' ')
 tokenizer.fit_on_texts(list(df_text))
sequences = tokenizer.texts_to_sequences(df_text)
 return sequences, tokenizer
def transform(df_text):
seqs, _ = tokenize(df_text)
 X = pad_sequences(seqs, maxlen=MAX_LEN, padding='pre', truncating='post')
 return X<load_from_csv>
ntrain = train_X.shape[0]
ntest = test_X.shape[0]
SEED = 0
NFOLDS = 5
kf = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state = SEED)
 class SklearnHelper(object):
def __init__(self, clf, seed=0, params=None):
params['random_state'] = seed
self.clf = clf(**params)
 def train(self, x_train, y_train):
self.clf.fit(x_train, y_train)
 def predict(self, x):
return self.clf.predict(x)
 def fit(self,x,y):
return self.clf.fit(x,y)
 def feature_importances(self,x,y):
print(self.clf.fit(x,y ).feature_importances_)
 def get_oof(clf, x_train, y_train, x_test):
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( NFOLDS, ntest))
fold = 0
for(train_index, test_index)in kf.split(x_train, y_train):
x_tr = x_train[train_index]
y_tr = y_train[train_index]
x_te = x_train[test_index]
clf.train(x_tr, y_tr)
 oof_train[test_index] = clf.predict(x_te)
 oof_test_skf[fold, :] = clf.predict(x_test)
 fold += 1
oof_test[:] = oof_test_skf.mean(axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)
class fakemodule(object):
@staticmethod
def transform(a):
return transform(a)
 dr = DataReader('%s/train.csv' % datapath, fakemodule, os.path.join(datapath, 'test.csv'))<load_pretrained>
rf_params = {
'n_jobs': -1,
'n_estimators': 500,
'warm_start': True,
'max_depth': 6,
'min_samples_leaf': 2,
'max_features' : 'sqrt',
'verbose': 0
}
et_params = {
'n_jobs': -1,
'n_estimators':500,
'max_depth': 8,
'min_samples_leaf': 2,
'verbose': 0
}
ada_params = {
'n_estimators': 500,
'learning_rate' : 0.75
}
gb_params = {
'n_estimators': 500,
'max_depth': 5,
'min_samples_leaf': 2,
'verbose': 0
}
svc_params = {
'kernel' : 'linear',
'C' : 0.025
}
rf = SklearnHelper(clf=RandomForestClassifier, seed=SEED, params=rf_params)
 et = SklearnHelper(clf=ExtraTreesClassifier, seed=SEED, params=et_params)
 ada = SklearnHelper(clf=AdaBoostClassifier, seed=SEED, params=ada_params)
 gb = SklearnHelper(clf=GradientBoostingClassifier, seed=SEED, params=gb_params)
 svc = SklearnHelper(clf=SVC, seed=SEED, params=svc_params)
 et_oof_train, et_oof_test = get_oof(et, train_X, train_Y, test_X)
 rf_oof_train, rf_oof_test = get_oof(rf, train_X, train_Y, test_X)
 ada_oof_train, ada_oof_test = get_oof(ada, train_X, train_Y, test_X)
 gb_oof_train, gb_oof_test = get_oof(gb, train_X, train_Y, test_X)
 svc_oof_train, svc_oof_test = get_oof(svc, train_X, train_Y, test_X)
 print("Training is complete")
 rf_feature = rf.feature_importances(train_X, train_Y)
 et_feature = et.feature_importances(train_X, train_Y)
 ada_feature = ada.feature_importances(train_X, train_Y)
 gb_feature = gb.feature_importances(train_X, train_Y )
t0 = time.time()
with timer("Extract Word Index From Train and Test Data"):
print("Loading data...")
 all_text = dr.get_all_text()
print('Tokenizing text...')
 _, tokenizer = tokenize(all_text)
 word_index = tokenizer.word_index
with timer("Create Embedding Weights Matrix"):
print('Loading embeddings file')
 word_embed = load_word_embedding(EMBED_FILEPATH)
 print('Create embeddings weights...')
 embed_weights = create_embedding_weights(
 word_index,
word_embed,
MAX_FEATURES)
 filepath_to_save = '{}.pkl'.format('glove')
 pd.to_pickle(embed_weights, filepath_to_save)
 print('Save embeddings weights to {}'.format(filepath_to_save))
print('Entire program is done and it took {:.2f}s'.format(time.time() - t0))<train_model>
x_train = np.concatenate(( et_oof_train, rf_oof_train, ada_oof_train, gb_oof_train, svc_oof_train), axis=1)
 x_test = np.concatenate(( et_oof_test, rf_oof_test, ada_oof_test, gb_oof_test, svc_oof_test), axis=1 )
<train_model><EOS> 
gbm = xgb.XGBClassifier(
 n_estimators= 2000,
max_depth= 4,
min_child_weight= 2,
gamma=0.9,
subsample=0.8,
colsample_bytree=0.8,
objective= 'binary:logistic',
nthread= -1,
scale_pos_weight=1 ).fit(train_X, train_Y)
 predictions = gbm.predict(test_X)
 submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": predictions
})
 submission.to_csv('submission.csv', index=False )