Kaggle competition: Titanic - Machine Learning from Disaster
x_test = test.values<predict_on_test>
make_scorer,classification_report,roc_auc_score,roc_curve,
average_precision_score,precision_recall_curve)
 pd.set_option('display.max_columns', None)
 warnings.filterwarnings("ignore")
 RANDOM_SEED = 101
x_test = scaler.transform(x_test)
 y_test = clf.predict_proba(x_test )<create_dataframe>
sub_file = pd.read_csv("/kaggle/input/titanic/gender_submission.csv")
 sub_file.head()
submission = pd.DataFrame(y_test, index=test_ids, columns=le.classes_ )<save_to_csv>
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 train.head()
submission.to_csv('./submission_leaf_classification.csv')
 print('Done' )<import_modules>
val = pd.read_csv("/kaggle/input/titanic/test.csv")
 val.head()
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from keras.models import Sequential
from keras.layers import Dense,Dropout,Activation
from keras.utils.np_utils import to_categorical
from keras.callbacks import EarlyStopping<categorify>
train.isnull().mean()
data = pd.read_csv('.. /input/leaf-classification/train.csv')
 parent_data = data.copy()
ID = data.pop('id')
 y = data.pop('species')
 y = LabelEncoder().fit(y ).transform(y)
 print(y.shape)
 X = StandardScaler().fit(data ).transform(data)
 print(X.shape)
 y_cat = to_categorical(y)
 print(y_cat.shape )<train_model>
target = 'Survived'
model = Sequential()
model.add(Dense(1500,input_dim=192, kernel_initializer = 'uniform', activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(1500, activation='sigmoid'))
model.add(Dropout(0.1))
model.add(Dense(99, activation='softmax'))
model.compile(loss='categorical_crossentropy',optimizer='rmsprop', metrics = ["accuracy"])
 early_stopping = EarlyStopping(monitor='val_loss', patience=280)
 history = model.fit(X,y_cat,batch_size=192,
epochs=800 ,verbose=0, validation_split=0.1, callbacks=[early_stopping])
<compute_test_metric>
def get_salutation_map(df,var,rare):
sal_dict = {}
for sal, count in df[var].value_counts().to_dict().items() :
count = int(count)
 if count < 10:
sal_dict[sal] = rare
else:
sal_dict[sal] = sal
return sal_dict
print("train/val loss ratio: ", min(history.history['loss'])/min(history.history['val_loss']))<save_to_csv>
train["Salutation"] = train["Name"].map(lambda x: x.split(',')[1].split() [0][:-1] )
test = pd.read_csv('.. /input/leaf-classification/test.csv')
 index = test.pop('id')
 test = StandardScaler().fit(test ).transform(test)
 yPred = model.predict_proba(test)
 yPred = pd.DataFrame(yPred,index=index,columns=sorted(parent_data.species.unique()))
fp = open('submission_nn_kernel.csv','w')
 fp.write(yPred.to_csv() )<split>
get_salutation_map(train,"Salutation","Rare" )
class Data_Clean(object):
def __init__(self):
self.numerical_data, self.num_test_data = self.read_numerical_data()
self.id, self.species, self.num_train, self.test_id, self.test_num = self.split_numerical_data()
def split_numerical_data(self):
id = self.numerical_data.pop('id')
 species = self.numerical_data.pop('species')
 species = LabelEncoder().fit(species ).transform(species)
 species = to_categorical(species, num_classes=99)
 self.numerical_data = StandardScaler().fit(self.numerical_data ).transform(self.numerical_data)
 test_id = self.num_test_data.pop('id')
 num_test_data = StandardScaler().fit(self.num_test_data ).transform(self.num_test_data)
 return id, species, self.numerical_data, test_id, num_test_data
def read_numerical_data(self):
root = ".. /input"
data = pd.read_csv('%s/train.csv' %root)
 test_data = pd.read_csv('%s/test.csv' %(root))
return data, test_data
def run(self):
return self.species, self.num_train, self.test_num, self.test_id
class CNN(object):
def __init__(self, num_train, num_vali, species_train, species_vali, num_test, test_id):
self.num_train = num_train
self.num_vali = num_vali
self.species_train = species_train
self.species_vali = species_vali
self.num_test = num_test
self.test_id = test_id
def define_CNN(self):
model = Sequential()
model.add(Dense(512, activation='relu', input_dim=192))
model.add(Dropout(0.25))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(99, activation='softmax'))
self.model = model
def RMSprop(self, batch_size, epoch):
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-8, decay=0.0)
 self.model.compile(optimizer=optimizer, loss = 'categorical_crossentropy', metrics=['accuracy'])
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', patience=3, verbose=1, factor=0.5, min_lr=0.00001)
 self.model.fit(self.num_train, self.species_train, batch_size=batch_size, epochs=epoch,
validation_data=(self.num_vali, self.species_vali), verbose=2, callbacks=[learning_rate_reduction])
 def make_predict(self):
ypred_prob = self.model.predict(self.num_test)
 root = ".. /input"
labels = sorted(pd.read_csv(os.path.join(root, 'train.csv')).species.unique())
 ypred = pd.DataFrame(ypred_prob, index = self.test_id, columns=labels)
 print(ypred.head(2))
fp = open('submit.csv', 'w')
 fp.write(ypred.to_csv())
 def run(self):
self.define_CNN()
self.RMSprop(batch_size = 128, epoch=100)
 self.make_predict()
if __name__ == '__main__':
data_clean = Data_Clean()
species, num, num_test, test_id = data_clean.run()
species_train, species_vali, num_train, num_vali = train_test_split(species, num, test_size=0.1, random_state=26)
 cnn = CNN(num_train, num_vali,species_train, species_vali, num_test, test_id)
 cnn.run() 
<import_modules>
train["Salutation"] = train["Name"].map(lambda x: x.split(',')[1].split() [0][:-1])
 train["Salutation"] = train["Salutation"].map(get_salutation_map(train,'Salutation','Rare'))
train.head(2 )
import csv as csv
import numpy as np
import pandas as pd
import matplotlib.cm as cm
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from scipy import stats
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import KFold,GridSearchCV 
<load_from_csv>
train['SibSp'].nunique()
traindf = pd.read_csv('.. /input/train.csv', header=0)
 x_train = traindf.drop(['id', 'species'], axis=1)
 y_train = traindf.pop('species')
 scaler = StandardScaler().fit(x_train)
 x_train = scaler.transform(x_train)
<choose_model_class>
train["Family_Size"].value_counts(normalize=True)*100
kfold = KFold(n_splits=5, shuffle=True, random_state=4)
<train_model>
def get_family_size_map(df,var):
fam_dict = {}
for size, pct in(df[var].value_counts(normalize=True)*100 ).to_dict().items() :
if size == 0:
fam_dict[size] = "Alone"
elif(size != 0)&(pct > 10.0):
fam_dict[size] = "Small"
else:
fam_dict[size] = "Large"
return fam_dict
rf = ExtraTreesClassifier(n_estimators=500, random_state=0)
 rf_validation=[rf.fit(x_train[train], y_train[train] ).score(x_train[test], y_train[test] ).mean() \
for train, test in kfold.split(x_train)]<prepare_x_and_y>
train["Family_Size"] = train["Family_Size"].map(get_family_size_map(train,'Family_Size'))
train.head(2 )
np.random.seed(42)
 train = pd.read_csv('.. /input/train.csv')
 x_train = train.drop(['id', 'species', 'margin7', 'margin15', 'margin33', 'texture14','margin51','margin60'], axis=1 ).values
le = LabelEncoder().fit(train['species'])
 y_train = le.transform(train['species'])
 scaler = StandardScaler().fit(x_train)
 x_train = scaler.transform(x_train)
 test = pd.read_csv('.. /input/test.csv')
 test = test.drop(['margin7', 'margin15', 'margin33', 'texture14','margin51','margin60'], axis=1)
 test_ids = test.pop('id')
 x_test = test.values
scaler = StandardScaler().fit(x_test)
 x_test = scaler.transform(x_test)
 log_reg = LogisticRegression(C=2200, multi_class="multinomial", tol=0.0001, solver='newton-cg')
 log_reg.fit(x_train, y_train)
 y_test = log_reg.predict_proba(x_test)
 submission = pd.DataFrame(y_test, index=test_ids, columns=le.classes_)
 submission.to_csv('submission.csv' )<load_from_csv>
train.isnull().mean()
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<split>
train['had_Cabin'] = np.where(train['Cabin'].isna() ,0,1 )
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<prepare_x_and_y>
train['Cabin'].dropna().map(lambda x:x[0] ).value_counts()
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]<import_modules>
train['Cabin'] = train['Cabin'].fillna("M")
 train['Cabin'] = train['Cabin'].map(lambda x: x[0] )
from keras.models import Sequential,Model
from keras.layers import Dense, CuDNNLSTM, Bidirectional, Input, Dropout, Embedding, CuDNNGRU, GlobalMaxPool1D
from keras.optimizers import Adam
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints<statistical_test>
train.groupby(['Salutation','had_Cabin'] )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
mean_dict = {}
for k, df in train.groupby(['Salutation','Family_Size','had_Cabin']):
if df['Age'].isnull().sum() != 0:
mean_dict[k] = df["Age"].mean()
mean_dict
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
for k,v in mean_dict.items() :
train.loc[(train["Salutation"] == k[0])&(train["Family_Size"] == k[1])&(train["had_Cabin"] == k[2])&(train["Age"].isna()), "Age"] = v
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<predict_on_test>
train['Embarked'].value_counts()
pred_cnn_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_cnn_val_y>thresh ).astype(int))))<predict_on_test>
train['Embarked'] = train['Embarked'].fillna(train['Embarked'].mode().values[0] )
pred_cnn_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
train.isnull().sum()
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<choose_model_class>
train.isnull().sum()
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<statistical_test>
num_cols = ['Age','Fare']
cat_cols = ['Pclass','Sex','Embarked','Cabin','had_Cabin','Salutation','Family_Size']
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 print(model.summary() )<train_model>
train_data = pd.get_dummies(train,columns=cat_cols,drop_first=True)
 train_data.head(2 )
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<predict_on_test>
explore_data, validation_data = train_test_split(train_data, test_size = 0.2, random_state=RANDOM_SEED, stratify=train[target] )
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))<predict_on_test>
train_data, test_data = train_test_split(explore_data, test_size = 0.2, random_state=RANDOM_SEED )
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
def handle_outliers_per_target_class(df,var,target,tol):
gdf = df[df[target] == 1]
var_data = gdf[var].values
q25, q75 = np.percentile(var_data, 25), np.percentile(var_data, 75)
 print('Outliers handling for {}'.format(var))
print('Quartile 25: {} | Quartile 75: {}'.format(q25, q75))
iqr = q75 - q25
print('IQR {}'.format(iqr))
cut_off = iqr * tol
lower, upper = q25 - cut_off, q75 + cut_off
print('Cut Off: {}'.format(cut_off))
print('{} Lower: {}'.format(var,lower))
print('{} Upper: {}'.format(var,upper))
outliers = [x for x in var_data if x < lower or x > upper]
print('Number of Outliers in feature {} in {}: {}'.format(var,key,len(outliers)))
 print('{} outliers:{}'.format(var,outliers))
print('----' * 25)
 print('
')
 print('
')
 return list(df[(df[var] > upper)|(df[var] < lower)].index )
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
outliers_wrt_target = []
for num_col in num_cols:
outliers_wrt_target.extend(handle_outliers_per_target_class(train_data,num_col,target,1.5))
outliers_wrt_target = list(set(outliers_wrt_target))
train_data = train_data.drop(outliers_wrt_target )
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 print(model.summary() )<train_model>
train_data["Fare"] = np.where(train_data["Fare"] != 0,np.log(train_data["Fare"]),np.log(0.00001))
test_data["Fare"] = np.where(test_data["Fare"] != 0,np.log(test_data["Fare"]),np.log(0.00001))
validation_data["Fare"] = np.where(validation_data["Fare"] != 0,np.log(validation_data["Fare"]),np.log(0.00001))
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<predict_on_test>
X_train = train_data.drop(['PassengerId', 'Survived', 'Name', 'SibSp', 'Parch', 'Ticket'],axis=1)
 y_train = train_data[target]
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))<predict_on_test>
X_test = test_data.drop(['PassengerId', 'Survived', 'Name', 'SibSp', 'Parch', 'Ticket'],axis=1)
 y_test = test_data[target]
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
X_val = validation_data.drop(['PassengerId', 'Survived', 'Name', 'SibSp', 'Parch', 'Ticket'],axis=1)
 y_val = validation_data[target]
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
y_enc = LabelEncoder()
y_train = y_enc.fit_transform(y_train)
 y_test = y_enc.transform(y_test)
 y_val = y_enc.transform(y_val )
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 print(model.summary() )<train_model>
sc = StandardScaler()
X_train[num_cols] = sc.fit_transform(X_train[num_cols])
 X_test[num_cols] = sc.transform(X_test[num_cols])
 X_val[num_cols] = sc.transform(X_val[num_cols] )
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<predict_on_test>
clf = LogisticRegression()
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))<predict_on_test>
clf.fit(X_train,y_train )
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
y_pred = clf.predict(X_test )
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<find_best_params>
confusion_matrix(y_test,y_pred )
pred_val_y =(4 * pred_glove_val_y + pred_fasttext_val_y + 3 * pred_paragram_val_y + 2 * pred_cnn_val_y)/ 10.0
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )<save_to_csv>
accuracy_score(y_test,y_pred )
pred_test_y =(4 * pred_glove_test_y + pred_fasttext_test_y + 3 * pred_paragram_test_y + 2 * pred_cnn_test_y)/ 10.0
pred_test_y =(pred_test_y > best_thresh ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<load_from_csv>
classification_models = ['LogisticRegression',
'SVC',
'DecisionTreeClassifier',
'RandomForestClassifier',
'AdaBoostClassifier']
def load_data() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 return train_df, test_df<split>
cm = []
acc = []
prec = []
rec = []
f1 = []
models = []
estimators = []
train_df, test_df = load_data()
train_df.sample()<compute_test_metric>
for classfication_model in classification_models:
model = eval(classfication_model )()
model.fit(X_train,y_train)
 y_pred = model.predict(X_test)
 models.append(type(model ).__name__)
 estimators.append(( type(model ).__name__,model))
cm.append(confusion_matrix(y_test,y_pred))
acc.append(accuracy_score(y_test,y_pred))
prec.append(precision_score(y_test,y_pred))
rec.append(recall_score(y_test,y_pred))
f1.append(f1_score(y_test,y_pred))
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32' )<load_pretrained>
vc = VotingClassifier(estimators)
 vc.fit(X_train,y_train )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
print('Found %s word vectors.' % len(embeddings_index))<feature_engineering>
y_pred = vc.predict(X_test)
 models.append(type(vc ).__name__)
 cm.append(confusion_matrix(y_test,y_pred))
acc.append(accuracy_score(y_test,y_pred))
prec.append(precision_score(y_test,y_pred))
rec.append(recall_score(y_test,y_pred))
f1.append(f1_score(y_test,y_pred))
def check_coverage(vocab,embeddings_index):
a, oov, k, i = {}, {}, 0, 0
for word in vocab:
try:
a[word] = embeddings_index[word]
k += vocab[word]
except:
oov[word] = vocab[word]
i += vocab[word]
pass
print(f'Found embeddings for {(len(a)/ len(vocab)) :.2%} of vocab')
 print(f'Found embeddings for {(k /(k + i)) :.2%} of all text')
 sorted_x = sorted(oov.items() , key=(lambda x: x[1]), reverse=True)
 return sorted_x
def get_vocab(question_series):
sentences = question_series.str.split().values
words = [item for sublist in sentences for item in sublist]
return dict(Counter(words))<split>
model_dict = {"Models":models,
"CM":cm,
"Accuracy":acc,
"Precision":prec,
"Recall":rec,
"f1_score":f1}
vocab = get_vocab(train_df["question_text"])
 out_of_vocab = check_coverage(vocab, embeddings_index)
 out_of_vocab[:10]<string_transform>
model_df = pd.DataFrame(model_dict)
 model_df
punct = set('?!.,"
embed_punct = punct & set(embeddings_index.keys())
 def clean_punctuation(txt):
for p in "/-":
txt = txt.replace(p, ' ')
 for p in "'`‘":
txt = txt.replace(p, '')
 for p in punct:
txt = txt.replace(p, f' {p} ' if p in embed_punct else ' _punct_ ')
 return txt<feature_engineering>
model_df.sort_values(by=['Accuracy','f1_score','Recall','Precision'],ascending=False,inplace=True)
 model_df
train_df["question_text"] = train_df["question_text"].map(lambda x: clean_punctuation(x)).str.replace('\d+', '
test_df["question_text"] = test_df["question_text"].map(lambda x: clean_punctuation(x)).str.replace('\d+', '
vocab = get_vocab(train_df["question_text"])
 out_of_vocab = check_coverage(vocab, embeddings_index )<string_transform>
model_param_grid = {}
maxlen = 65
max_features = 60000
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=201901)
 train_X = train_df["question_text"].fillna("_
val_X = val_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<define_variables>
model_param_grid['LogisticRegression'] = {'penalty' : ['l1', 'l2'],
'C' : np.logspace(0, 4, 10)}
def prepare_embedding_matrix(embeddings_index,word_index,num_words):
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(num_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix<categorify>
model_param_grid['SVC'] = [{'kernel': ['rbf'],
'gamma': [1e-2, 1e-3, 1e-4, 1e-5],
'C': [0.001, 0.10, 0.1, 10, 25, 50, 100, 1000]},
{'kernel': ['sigmoid'],
'gamma': [1e-2, 1e-3, 1e-4, 1e-5],
'C': [0.001, 0.10, 0.1, 10, 25, 50, 100, 1000]},
{'kernel': ['linear'],
'C': [0.001, 0.10, 0.1, 10, 25, 50, 100, 1000]},
{'kernel': ['poly'],
'degree' : [0, 1, 2, 3, 4, 5, 6]}
]
EMBEDDING_DIM = 300
word_index = tokenizer.word_index
num_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = prepare_embedding_matrix(embeddings_index,word_index,num_words )<set_options>
model_param_grid['DecisionTreeClassifier'] = {'criterion' : ["gini","entropy"],
'max_features': ['auto', 'sqrt', 'log2'],
'min_samples_split': [10,11,12,13,14,15],
'min_samples_leaf':[1,2,3,4,5,6,7]}
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
model_param_grid['RandomForestClassifier'] = {'n_estimators' : [50,100,150,200],
'criterion' : ["gini","entropy"],
'max_features': ['auto', 'sqrt', 'log2'],
'class_weight' : ["balanced", "balanced_subsample"]}
inp = Input(shape=(maxlen,))
x = Embedding(max_features, EMBEDDING_DIM, weights=[embedding_matrix],trainable=False )(inp)
 x = SpatialDropout1D(0.25 )(x)
 x1 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 attn_lstm = Attention(maxlen )(x1)
 attn_gru = Attention(maxlen )(x2)
 concat = Concatenate()([attn_lstm,attn_gru])
 concat = BatchNormalization()(concat)
 d = Dense(256, activation="relu" )(concat)
 d = Dropout(0.3 )(d)
 out = Dense(1, activation="sigmoid" )(d)
 model = Model(inputs=inp, outputs=out)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<train_model>
model_param_grid['AdaBoostClassifier'] = {'n_estimators' : [25,50,75,100],
'learning_rate' : [0.001,0.01,0.05,0.1,1,10],
'algorithm' : ['SAMME', 'SAMME.R']}
model.fit(train_X, train_y, batch_size=512, epochs=4, validation_data=(val_X, val_y))<predict_on_test>
def tune_parameters(model_name,model,params,cv,scorer,X,y):
best_model = GridSearchCV(estimator = model,
param_grid = params,
scoring = scorer,
cv = cv,
n_jobs = -1 ).fit(X, y)
 print("Tuning Results for ", model_name)
 print("Best Score Achieved: ",best_model.best_score_)
 print("Best Parameters Used: ",best_model.best_params_)
 return best_model.best_estimator_
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1 )<compute_test_metric>
def roc_metric(y_test, y_pred):
score = roc_auc_score(y_test, y_pred)
 return score
result = threshold_search(val_y, pred_glove_val_y)
 print(result )<save_model>
roc_scorer = make_scorer(roc_metric,greater_is_better=True )
model.save('my_model.h5')
<predict_on_test>
best_estimators = []
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<save_to_csv>
for m_name, m_obj in estimators:
best_estimators.append(( m_name,tune_parameters(m_name,
m_obj,
model_param_grid[m_name],
10,
roc_scorer,
X_train,
y_train)) )
pred_test_y = pred_glove_test_y
pred_test_y =(pred_test_y > result['threshold'] ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<import_modules>
best_estimators
tqdm.pandas(desc='Progress')
<define_variables>
tuned_vc = VotingClassifier(best_estimators)
 tuned_vc.fit(X_train,y_train )
embed_size = 300
max_features = 120000
maxlen = 80
batch_size = 512
n_epochs = 5
n_splits = 5
debug = 0
num_embeddings = 2<define_variables>
y_pred = tuned_vc.predict(X_test )
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ] 
<define_variables>
confusion_matrix(y_test,y_pred )
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"that's" : "that is",
"there's" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}<define_variables>
accuracy_score(y_test,y_pred )
first_word_mispell_dict = {
'whta': 'what', 'howdo': 'how do', 'Whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much',
'howmany': 'how many', 'whydo': 'why do', 'doi': 'do i', 'howdoes': 'how does', "whst": 'what',
'shoupd': 'should', 'whats': 'what is', "im": "i am", "whatis": "what is", "iam": "i am", "wat": "what",
"wht": "what","whts": "what is", "whtwh": "what", "whtat": "what", "whtlat": "what", "dueto to": "due to",
"dose": "does", "wha": "what", 'hw': "how", "its": "it is", "whay": "what", "ho": "how", "whart": "what",
"woe": "wow", "wt": "what", "ive": "i have","wha": "what", "wich": "which", "whic": "which", "whys": "why",
"doe": "does", "wjy": "why", "wgat": "what", "hiw": "how","howto": "how to", "lets": "let us", "haw": "how",
"witch": "which", "wy": "why", "girlfriend": "girl friend", "hows": "how is","whyis": "why is", "whois": "who is",
"dont": "do not", "hat": "what", "whos": "who is", "whydoes": "why does", "whic": "which","hy": "why", "w? hy": "why",
"ehat": "what", "whate": "what", "whai": "what", "whichis": "which is", "whi": "which", "isit": "is it","ca": "can",
"wwhat": "what", "wil": "will", "wath": "what", "plz": "please", "ww": "how", "hou": "how", "whch": "which",
"ihave": "i have", "cn": "can", "doesnt": "does not", "shoul": "should", "whatdo": "what do", "isnt": "is not",
"whare": "what are","whick": "which", "whatdoes": "what does", "hwo": "how", "howdid": "how did", "why dose": "why does"
}<categorify>
precision_score(y_test,y_pred )
def clean_text(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
if bool(re.search(r'\d', x)) :
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x 
<string_transform>
recall_score(y_test,y_pred )
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def correct_first_word(x):
for key in first_word_mispell_dict.keys() :
if x.startswith(key + " "):
x = x.replace(key + " ", first_word_mispell_dict[key] + " ")
 break
return x<feature_engineering>
f1_score(y_test,y_pred )
def add_features(df):
df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))
df["lower_question_text"] = df["question_text"].apply(lambda x: x.lower())
 df['total_length'] = df['question_text'].progress_apply(len)
 df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df<concatenate>
import keras
from keras.utils import plot_model
from keras.models import Model,Sequential,load_model
from keras.layers import Input, Flatten, Dense, Dropout
from keras.layers.merge import concatenate
from keras import backend as K
from keras.callbacks import ModelCheckpoint,EarlyStopping,ReduceLROnPlateau
def parallelize_apply(df,func,colname,num_process,newcolnames):
pool =Pool(processes=num_process)
 arraydata = pool.map(func,tqdm(df[colname].values))
pool.close()
newdf = pd.DataFrame(arraydata,columns = newcolnames)
 df = pd.concat([df,newdf],axis=1)
 return df
def parallelize_dataframe(df, func):
df_split = np.array_split(df, 4)
 pool = Pool(4)
 df = pd.concat(pool.map(func, df_split))
pool.close()
pool.join()
return df<load_from_csv>
def nn_model(X,y,optimizer,kernels):
input_shape = X.shape[1]
if(len(np.unique(y)) == 2):
op_neurons = 1
op_activation = 'sigmoid'
loss = 'binary_crossentropy'
else:
op_neurons = len(np.unique(y))
op_activation = 'softmax'
loss = 'categorical_crossentropy'
classifier = Sequential()
classifier.add(Dense(units = input_shape,
kernel_initializer = kernels,
activation = 'relu',
input_dim = input_shape))
classifier.add(Dense(units = 8,
kernel_initializer = kernels,
activation = 'relu'))
classifier.add(Dense(units = 4,
kernel_initializer = kernels,
activation = 'relu'))
classifier.add(Dropout(rate = 0.25))
classifier.add(Dense(units = op_neurons,
kernel_initializer = kernels,
activation = op_activation))
classifier.compile(optimizer = optimizer,
loss = loss,
metrics = ['accuracy'])
 classifier.summary()
return classifier
if debug:
train_df = pd.read_csv(".. /input/quora-insincere-questions-classification/train.csv")[:800]
test_df = pd.read_csv(".. /input/quora-insincere-questions-classification/test.csv")[:200]
else:
train_df = pd.read_csv(".. /input/quora-insincere-questions-classification/train.csv")
 test_df = pd.read_csv(".. /input/quora-insincere-questions-classification/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<create_dataframe>
model = nn_model(X_train,y_train,'adam','he_uniform')
 history = model.fit(X_train,
y_train,
batch_size = 64,
epochs = 1000,
validation_data=(X_test, y_test))
train = parallelize_dataframe(train_df, add_features)
 test = parallelize_dataframe(test_df, add_features )<feature_engineering>
his_df = pd.DataFrame(history.history)
 his_df.shape