Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<feature_engineering>
print(os.listdir(".. /input"))
data_train['Age_NA'] =np.where(data_train.Age.isnull() , 1, 0)
 data_test['Age_NA'] =np.where(data_test.Age.isnull() , 1, 0 )<feature_engineering>
df=pd.read_csv('.. /input/train.csv')
 dftest=pd.read_csv('.. /input/test.csv')
 print(df.shape, dftest.shape)
data_train['Age_mean'] =np.where(data_train.Age.isnull() , data_train['Age'].mean() , data_train['Age'])
 data_test['Age_mean'] =np.where(data_test.Age.isnull() , data_test['Age'].mean() , data_test['Age'])
<count_unique_values>
train_df, val_df = train_test_split(df, test_size=0.1, random_state=1995)
for column in data_train.columns:
print(column, len(data_train[column].unique()))<drop_column>
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = dftest["question_text"].fillna("_na_" ).values
data_train = data_train.drop(['PassengerId'], axis=1)
 data_test = data_test.drop(['PassengerId'], axis=1 )<string_transform>
embed_size = 300
max_features = 95000
maxlen = 70
def ticket_sep(data_ticket):
ticket_type = []
for i in range(len(data_ticket)) :
ticket =data_ticket.iloc[i]
for c in string.punctuation:
ticket = ticket.replace(c,"")
 splited_ticket = ticket.split(" ")
 if len(splited_ticket)== 1:
ticket_type.append('NO')
 else:
ticket_type.append(splited_ticket[0])
 return ticket_type<feature_engineering>
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X )
data_train["ticket_type"] = ticket_sep(data_train.Ticket)
 data_train.head()<feature_engineering>
train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen )
data_test["ticket_type"]= ticket_sep(data_test.Ticket)
 data_test.head()<feature_engineering>
train_y = train_df['target'].values
val_y = val_df['target'].values
data_train["ticket_type"] = np.where(data_train["ticket_type"]=='SOTONOQ', 'A5', data_train["ticket_type"])
 data_test["ticket_type"] = np.where(data_test["ticket_type"]=='SOTONOQ', 'A5', data_test["ticket_type"])
<drop_column>
from sklearn.feature_extraction.text import TfidfVectorizer
data_train = data_train.drop(['Ticket'], axis=1)
 data_test = data_test.drop(['Ticket'], axis=1 )<count_missing_values>
vectorizer = TfidfVectorizer()
print('Missing values in Train set:', data_train.Cabin.isnull().sum())
 print('Missing values in Test set:', data_test.Cabin.isnull().sum() )<concatenate>
def attention_3d_block(inputs):
TIME_STEPS = inputs.shape[1].value
SINGLE_ATTENTION_VECTOR = False
input_dim = int(inputs.shape[2])
 a = Permute(( 2, 1))(inputs)
 a = Reshape(( input_dim, TIME_STEPS))(a)
 a = Dense(TIME_STEPS, activation='softmax' )(a)
 if SINGLE_ATTENTION_VECTOR:
a = Lambda(lambda x: K.mean(x, axis=1))(a)
 a = RepeatVector(input_dim )(a)
 a_probs = Permute(( 2, 1))(a)
 output_attention_mul = Multiply()([inputs, a_probs])
 return output_attention_mul
def cabin_sep(data_cabin):
cabin_type = []
for i in range(len(data_cabin)) :
if data_cabin.isnull() [i] == True:
cabin_type.append('NaN')
 else:
cabin = data_cabin[i]
cabin_type.append(cabin[:1])
 return cabin_type<feature_engineering>
class AttLayer(Layer):
def __init__(self, attention_dim):
self.init = initializers.get('normal')
 self.supports_masking = True
self.attention_dim = attention_dim
super(AttLayer, self ).__init__()
def build(self, input_shape):
assert len(input_shape)== 3
self.W = K.variable(self.init(( input_shape[-1], self.attention_dim)))
 self.b = K.variable(self.init(( self.attention_dim,)))
 self.u = K.variable(self.init(( self.attention_dim, 1)))
 self.trainable_weights = [self.W, self.b, self.u]
super(AttLayer, self ).build(input_shape)
 def compute_mask(self, inputs, mask=None):
return mask
def call(self, x, mask=None):
uit = K.tanh(K.bias_add(K.dot(x, self.W), self.b))
ait = K.dot(uit, self.u)
 ait = K.squeeze(ait, -1)
 ait = K.exp(ait)
 if mask is not None:
ait *= K.cast(mask, K.floatx())
 ait /= K.cast(K.sum(ait, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 ait = K.expand_dims(ait)
 weighted_input = x * ait
output = K.sum(weighted_input, axis=1)
 return output
def compute_output_shape(self, input_shape):
return(input_shape[0], input_shape[-1] )
data_train['cabin_type'] = cabin_sep(data_train.Cabin)
 data_test['cabin_type'] = cabin_sep(data_test.Cabin)
 data_train.head()<drop_column>
import gc
data_train = data_train.drop(['Cabin'], axis=1)
 data_test = data_test.drop(['Cabin'], axis=1 )<string_transform>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_1 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_1[i] = embedding_vector
del embeddings_index; gc.collect()
def name_sep(data):
families=[]
titles = []
new_name = []
for i in range(len(data)) :
name = data.iloc[i]
if '(' in name:
name_no_bracket = name.split('(')[0]
else:
name_no_bracket = name
family = name_no_bracket.split(",")[0]
title = name_no_bracket.split(",")[1].strip().split(" ")[0]
for c in string.punctuation:
name = name.replace(c,"" ).strip()
family = family.replace(c,"" ).strip()
title = title.replace(c,"" ).strip()
families.append(family)
 titles.append(title)
 new_name.append(name)
 return families, titles, new_name<feature_engineering>
def model2() :
inp = Input(shape=(maxlen,))
embed = Embedding(max_features, embed_size, weights=[embedding_matrix_1], trainable=False )(inp)
 x = embed
x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = attention_3d_block(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = AttLayer(64 )(x)
 x = Dropout(0.3 )(x)
 x = Dense(128, activation='relu' )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
data_train['family'], data_train['title'], data_train['Name'] = name_sep(data_train.Name)
 data_test['family'], data_test['title'], data_test['Name'] = name_sep(data_test.Name)
 data_train.head()<count_unique_values>
from keras.models import Model
from keras.layers import Input, Dense, Embedding, concatenate
from keras.layers import CuDNNGRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D
from keras.layers import Add, BatchNormalization, Activation, CuDNNLSTM, Dropout
from keras.layers import *
from keras.models import *
from keras.preprocessing import text, sequence
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
import gc
from sklearn import metrics
len([x for x in data_train.family.unique() if x in data_test.family.unique() ] )<count_unique_values>
F1scores_epochs = [0.6327, 0.6607, 0.6688]
len([x for x in data_train.family.unique() if x not in data_test.family.unique() ] )<count_unique_values>
F1scores_epochs = [0.6327, 0.6607, 0.6688, 0.6698]
len([x for x in data_test.family.unique() if x not in data_train.family.unique() ] )<feature_engineering>
MODEL2 = model2()
MODEL2.summary()
batch_size = 2048
epochs = 5
early_stopping = EarlyStopping(patience=3, verbose=1, monitor='val_loss', mode='min')
 model_checkpoint = ModelCheckpoint('./model2.model', save_best_only=True, verbose=1, monitor='val_loss', mode='min')
 reduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)
 hist = MODEL2.fit(train_X, train_y, batch_size=batch_size, epochs=epochs, validation_data=(val_X, val_y), verbose=True)
 MODEL2.save('./model2.h5' )
data_train['family_size'] = data_train.SibSp + data_train.Parch +1
data_test['family_size'] = data_test.SibSp + data_test.Parch +1
rate_family = data_train.groupby('family')['Survived', 'family','family_size'].median()
rate_family.head()<feature_engineering>
pred_val_y_2 = MODEL2.predict([val_X], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y_2 > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh_2 = thresholds[0][0]
print("Best threshold: ", best_thresh_2)
 y_pred_2 = MODEL2.predict(test_X, batch_size=1024, verbose=True)
 print(y_pred_2 )
<feature_engineering><EOS> 
y_te =(y_pred_2[:,0] > best_thresh_2 ).astype(np.int)
 submit_df = pd.DataFrame({"qid": dftest["qid"], "prediction": y_te})
 submit_df.to_csv("submission.csv", index=False)