Kaggle competition: Quora Insincere Questions Classification
train = titanic_dummies[ :train_idx]
test = titanic_dummies[test_idx: ]
train.Survived = train.Survived.astype(int )<prepare_x_and_y>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )
X = train.drop('Survived', axis=1 ).values
y = train.Survived.values<drop_column>
X_train = train_df["question_text"].fillna("na" ).values
X_test = test_df["question_text"].fillna("na" ).values
y = train_df["target"]
X_test = test.drop('Survived', axis=1 ).values 
<define_search_space>
from keras.models import Model
from keras.layers import Input, Dense, Embedding, concatenate
from keras.layers import CuDNNGRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D
from keras.layers import Add, BatchNormalization, Activation, CuDNNLSTM, Dropout
from keras.layers import *
from keras.models import *
from keras.optimizers import *
from keras.preprocessing import text, sequence
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
import gc
from sklearn import metrics
from keras import regularizers
log_params = dict(
 C = np.logspace(-5, 8, 15),
penalty = ['l1', 'l2']
)<train_on_grid>
maxlen = 50
max_features = 50000
embed_size = 300
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X_train)+ list(X_test))
X_train = tokenizer.texts_to_sequences(X_train)
 X_test = tokenizer.texts_to_sequences(X_test)
 x_train = sequence.pad_sequences(X_train, maxlen=maxlen)
 x_test = sequence.pad_sequences(X_test, maxlen=maxlen )
log = LogisticRegression()
logreg_cv = GridSearchCV(estimator=log, param_grid=log_params, cv=5)
 logreg_cv.fit(X, y )<train_model>
def attention_3d_block(inputs):
TIME_STEPS = inputs.shape[1].value
SINGLE_ATTENTION_VECTOR = False
input_dim = int(inputs.shape[2])
 a = Permute(( 2, 1))(inputs)
 a = Reshape(( input_dim, TIME_STEPS))(a)
 a = Dense(TIME_STEPS, activation='softmax' )(a)
 if SINGLE_ATTENTION_VECTOR:
a = Lambda(lambda x: K.mean(x, axis=1))(a)
 a = RepeatVector(input_dim )(a)
 a_probs = Permute(( 2, 1))(a)
 output_attention_mul = Multiply()([inputs, a_probs])
 return output_attention_mul
print("Tuned Logistic Regression Parameters: {}".format(logreg_cv.best_params_))
print("Best score is {}".format(logreg_cv.best_score_))<predict_on_test>
class AttLayer(Layer):
def __init__(self, attention_dim):
self.init = initializers.get('normal')
 self.supports_masking = True
self.attention_dim = attention_dim
super(AttLayer, self ).__init__()
def build(self, input_shape):
assert len(input_shape)== 3
self.W = K.variable(self.init(( input_shape[-1], self.attention_dim)))
 self.b = K.variable(self.init(( self.attention_dim,)))
 self.u = K.variable(self.init(( self.attention_dim, 1)))
 self.trainable_weights = [self.W, self.b, self.u]
super(AttLayer, self ).build(input_shape)
 def compute_mask(self, inputs, mask=None):
return mask
def call(self, x, mask=None):
uit = K.tanh(K.bias_add(K.dot(x, self.W), self.b))
ait = K.dot(uit, self.u)
 ait = K.squeeze(ait, -1)
 ait = K.exp(ait)
 if mask is not None:
ait *= K.cast(mask, K.floatx())
 ait /= K.cast(K.sum(ait, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 ait = K.expand_dims(ait)
 weighted_input = x * ait
output = K.sum(weighted_input, axis=1)
 return output
def compute_output_shape(self, input_shape):
return(input_shape[0], input_shape[-1] )
log_pred = logreg_cv.predict(X_test )<define_search_space>
def model_cnn(embedding_matrix):
filter_sizes = [1, 2, 3, 5]
num_filters = 64
inp = Input(shape=(maxlen,))
embed = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.1 )(embed)
 mpool = []
x = Reshape(( maxlen, embed_size, 1))(x)
 for fil in filter_sizes:
conv = Conv2D(num_filters,(fil, embed_size), kernel_initializer='he_normal', activation='relu' )(x)
 pool = MaxPool2D(pool_size=(maxlen - fil + 1, 1))(conv)
 mpool.append(pool)
 x = Concatenate(axis=1 )(mpool)
 x = Flatten()(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inp, x)
 return model
def model_bgru(embedding_matrix):
inp = Input(shape=(maxlen,))
embed = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = embed
x = SpatialDropout1D(0.2 )(x)
 x1 = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x1)
 x = Add()([x1, x2])
 x = AttLayer(maxlen )(x)
 x = Dropout(0.4 )(x)
 x = Dense(1, activation='sigmoid' )(x)
 model = Model(inp, x)
 return model
forrest_params = dict(
 max_depth = [n for n in range(9, 14)],
min_samples_split = [n for n in range(4, 11)],
min_samples_leaf = [n for n in range(2, 5)],
n_estimators = [n for n in range(10, 60, 10)],
)<choose_model_class>
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42 )
forrest = RandomForestClassifier()<train_on_grid>
VAL_Y = []
TEST_Y = []
forest_cv = GridSearchCV(estimator=forrest, param_grid=forrest_params, cv=5)
 forest_cv.fit(X, y )<find_best_params>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
print("Best score: {}".format(forest_cv.best_score_))
print("Optimal params: {}".format(forest_cv.best_estimator_))<predict_on_test>
model = model_cnn(embedding_matrix)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 batch_size = 2048
epochs = 5
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 model.save('./model_cnn_glove.h5' )
forrest_pred = forest_cv.predict(X_test)
 print(forrest_pred )<create_dataframe>
val_pred_cnn_glove = model.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(val_pred_cnn_glove > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 y_pred_cnn_glove = model.predict(x_test, batch_size=1024, verbose=True)
 VAL_Y.append(val_pred_cnn_glove)
 TEST_Y.append(y_pred_cnn_glove )
kaggle = pd.DataFrame({'PassengerId': passengerId, 'Survived': forrest_pred} )<save_to_csv>
model = model_bgru(embedding_matrix)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 batch_size = 2048
epochs = 5
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 model.save('./model_bgru_glove.h5' )
filename = 'submit.csv'
kaggle.to_csv(filename, index=False )<load_from_csv>
val_pred_bgru_glove = model.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(val_pred_bgru_glove > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 y_pred_bgru_glove = model.predict(x_test, batch_size=1024, verbose=True)
 VAL_Y.append(val_pred_bgru_glove)
 TEST_Y.append(y_pred_bgru_glove )
train=pd.read_csv('.. /input/train.csv')
 test=pd.read_csv('.. /input/test.csv' )<drop_column>
del embeddings_index;
del embedding_matrix
gc.collect()
train.drop(labels='Cabin',inplace=True,axis=1)
 test.drop(labels='Cabin',inplace=True,axis=1)
<feature_engineering>
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
def check_class(x):
if pd.isnull(x['Age']):
return pmean[x['Pclass']]
return x['Age']
pmean=train.groupby('Pclass' ).mean() ['Age']
train['Age']=train.apply(check_class,axis=1)
 test['Age']=test.apply(check_class,axis=1)
<data_type_conversions>
model = model_bgru(embedding_matrix)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 batch_size = 2048
epochs = 5
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 model.save('./model_bgru_wiki.h5' )
test['Fare']=test['Fare'].fillna(np.mean(test['Fare'])).astype(float)
<prepare_x_and_y>
val_pred_bgru_wiki = model.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(val_pred_bgru_wiki > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 y_pred_bgru_wiki = model.predict(x_test, batch_size=1024, verbose=True)
 VAL_Y.append(val_pred_bgru_wiki)
 TEST_Y.append(y_pred_bgru_wiki )
vtrain=train
vtest=test<filter>
del embeddings_index;
del embedding_matrix
gc.collect()
train=train.loc[:,['Pclass','Survived','Sex','Age','SibSp','Parch','Fare','Embarked']]
test=test.loc[:,['Pclass','Sex','Age','SibSp','Parch','Fare','Embarked']]<categorify>
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
pc=pd.get_dummies(train['Pclass'],drop_first=True,prefix='pclass')
 pctest=pd.get_dummies(test['Pclass'],drop_first=True,prefix='pclass')
 sex=pd.get_dummies(train['Sex'],drop_first=True,prefix='sex')
 sextest=pd.get_dummies(test['Sex'],drop_first=True,prefix='sex')
 em=pd.get_dummies(train['Embarked'],drop_first=True)
 emtest=pd.get_dummies(test['Embarked'],drop_first=True )<categorify>
model = model_bgru(embedding_matrix)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 batch_size = 2048
epochs = 5
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 model.save('./model_bgru_paragram.h5' )
def fill(x):
for i in range(len(unique_surnames_train)) :
if unique_surnames_train[i] in x:
return i
extratrain=pd.get_dummies(vtrain["Name"].apply(fill ).replace(np.arange(len(unique_surnames_train)) ,unique_surnames_train),drop_first=True)
 extratest=pd.get_dummies(vtest["Name"].apply(fill ).replace(np.arange(len(unique_surnames_train)) ,unique_surnames_train),drop_first=True)
<concatenate>
val_pred_bgru_paragram = model.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(val_pred_bgru_paragram > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 y_pred_bgru_paragram = model.predict(x_test, batch_size=1024, verbose=True)
 VAL_Y.append(val_pred_bgru_paragram)
 TEST_Y.append(y_pred_bgru_paragram )
temp=pd.concat([extratest,pd.DataFrame(np.zeros(( test.shape[0],9)) ,columns=['Col.','Don.','Jonkheer.','Lady.','Major.','Mlle.','Mme.','Sir.', 'the'] ).astype('int')],axis=1)
<categorify>
pred_val_y =(2.5*val_pred_cnn_glove + 2.5*val_pred_bgru_glove + 2.5*val_pred_bgru_wiki + 2.5*val_pred_bgru_paragram)/10
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )
def hasalpha(x):
for i in x:
if str.isalpha(i):
return i
return 'non'
extr=pd.get_dummies(vtrain['Ticket'].apply(hasalpha)).iloc[:,:-1]
exte=pd.get_dummies(vtest['Ticket'].apply(hasalpha)).iloc[:,:-1].loc[:,extr.columns]<concatenate>
y_pred =(2.5*y_pred_cnn_glove + 2.5*y_pred_bgru_glove + 2.5*y_pred_bgru_wiki + 2.5*y_pred_bgru_paragram)/10
y_te =(y_pred[:,0] > best_thresh ).astype(np.int)
 submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": y_te})
 submit_df.to_csv("submission.csv", index=False )
<load_from_csv><EOS> 
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv(index =False)
 b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html)
 create_download_link(submit_df )