Kaggle competition: Quora Insincere Questions Classification
print("Unique Keywords : {}, Unique locations : {}".format(len(train.keyword.unique()), len(train.location.unique())))
 train.keyword.unique()<train_model>
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english")
print("Train :
")
 print(train.isnull().sum())
 print("
Test :
")
 print(test.isnull().sum() )<drop_column>
gc.collect()
K.clear_session()
ydata = train["target"]
train = train.drop(["target","location", "id"], axis=1)
 test = test.drop(["id","location"], axis=1 )<categorify>
class AttentionWeightedAverage(Layer):
def __init__(self, return_attention=False, **kwargs):
self.init = initializers.get('uniform')
 self.supports_masking = True
self.return_attention = return_attention
super(AttentionWeightedAverage, self ).__init__(** kwargs)
 def build(self, input_shape):
self.input_spec = [InputSpec(ndim=3)]
assert len(input_shape)== 3
self.W = self.add_weight(shape=(input_shape[2], 1),
name='{}_W'.format(self.name),
initializer=self.init)
 self.trainable_weights = [self.W]
super(AttentionWeightedAverage, self ).build(input_shape)
 def call(self, x, mask=None):
logits = K.dot(x, self.W)
 x_shape = K.shape(x)
 logits = K.reshape(logits,(x_shape[0], x_shape[1]))
ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))
if mask is not None:
mask = K.cast(mask, K.floatx())
 ai = ai * mask
att_weights = ai /(K.sum(ai, axis=1, keepdims=True)+ K.epsilon())
 weighted_input = x * K.expand_dims(att_weights)
 result = K.sum(weighted_input, axis=1)
 if self.return_attention:
return [result, att_weights]
return result
def get_output_shape_for(self, input_shape):
return self.compute_output_shape(input_shape)
 def compute_output_shape(self, input_shape):
output_len = input_shape[2]
if self.return_attention:
return [(input_shape[0], output_len),(input_shape[0], input_shape[1])]
return(input_shape[0], output_len)
 def compute_mask(self, input, input_mask=None):
if isinstance(input_mask, list):
return [None] * len(input_mask)
 else:
return None
keyword_mapper = {}
temp = train.keyword.unique()
for i in range(len(temp)) :
keyword_mapper[temp[i]] = i
train["keyword"] = train["keyword"].map(keyword_mapper)
 test["keyword"] = test["keyword"].map(keyword_mapper )<feature_engineering>
spell_model = gensim.models.KeyedVectors.load_word2vec_format('.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec')
 words = spell_model.index2word
w_rank = {}
for i,word in enumerate(words):
w_rank[word] = i
WORDS = w_rank
def text_preprocessing(data):
data = data.apply(lambda x: x.strip().lower())
 data = data.apply(lambda x: re.sub(r'\d+', '', x))
data = data.apply(lambda x: x.translate(str.maketrans('', '', string.punctuation)))
 data = data.apply(lambda x : word_tokenize(x))
data = data.apply(lambda x: [word for word in x if word not in stop_words])
 lemmatizer = WordNetLemmatizer()
data = data.apply(lambda x: [lemmatizer.lemmatize(word, pos ='v')for word in x])
 return data 
<feature_engineering>
del spell_model, w_rank
gc.collect()
test["text"] = text_preprocessing(test["text"])
 train["text"] = text_preprocessing(train["text"] )<categorify>
def words(text): return re.findall(r'\w+', text.lower())
 def P(word):
"Probability of `word`."
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def edits2(word):
"All edits that are two edits away from `word`."
return(e2 for e1 in edits1(word)for e2 in edits1(e1))
def singlify(word):
return "".join([letter for i,letter in enumerate(word)if i == 0 or letter != word[i-1]] )
vectorizer = TfidfVectorizer()
vector = vectorizer.fit_transform(["".join(i)for i in train["text"]])
 vector = vector.todense()
vector = np.concatenate(( vector, np.reshape(np.array(train["keyword"]),(train.keyword.shape[0],-1))), axis=1)
 print(vector.shape)
 vector_test = vectorizer.transform(["".join(i)for i in test["text"]])
 vector_test = vector_test.todense()
vector_test = np.concatenate(( vector_test, np.reshape(np.array(test["keyword"]),(test.keyword.shape[0],-1))), axis=1)
 print(vector_test.shape )<split>
def load_glove(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words
xtrain, xtest, ytrain, ytest = train_test_split(vector, ydata, train_size = 0.75 )<compute_test_metric>
def load_fasttext(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words
model = LinearSVC(loss="hinge",fit_intercept=False, max_iter=1500)
 model = model.fit(xtrain, ytrain)
 predictions = model.predict(xtest)
 print("Accuracy score: ", accuracy_score(ytest, predictions))
print("Precision score: ", precision_score(ytest, predictions))
print("Recall score: ", recall_score(ytest, predictions))
print("F1 score : ", f1_score(predictions, ytest))
confusion_matrix(ytest, predictions )<compute_test_metric>
def load_para(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words
model_lr = LogisticRegression(penalty='l2')
 model_lr = model_lr.fit(xtrain, ytrain)
 predictions = model_lr.predict(xtest)
 print("Accuracy score: ", accuracy_score(ytest, predictions))
print("Precision score: ", precision_score(ytest, predictions))
print("Recall score: ", recall_score(ytest, predictions))
print("F1 score : ", f1_score(predictions, ytest))
confusion_matrix(ytest, predictions )<compute_test_metric>
def build_model(embedding_matrix, nb_words, embedding_size=300):
inp = Input(shape=(max_length,))
x = Embedding(nb_words, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.3 )(x)
 x1 = Bidirectional(LSTM(256, return_sequences=True))(x)
 x2 = Bidirectional(GRU(128, return_sequences=True))(x1)
 max_pool1 = GlobalMaxPooling1D()(x1)
 max_pool2 = GlobalMaxPooling1D()(x2)
 conc = Concatenate()([max_pool1, max_pool2])
 predictions = Dense(1, activation='sigmoid' )(conc)
 model = Model(inputs=inp, outputs=predictions)
 adam = optimizers.Adam(lr=learning_rate)
 model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])
 return model
model_xgb = XGBClassifier(metric= 'euclidean', n_neighbors= 3, weights= 'uniform')
 model_xgb = model_xgb.fit(xtrain, ytrain)
 predictions = model_xgb.predict(xtest)
 print("Accuracy score: ", accuracy_score(ytest, predictions))
print("Precision score: ", precision_score(ytest, predictions))
print("Recall score: ", recall_score(ytest, predictions))
print("F1 score : ", f1_score(predictions, ytest))
confusion_matrix(ytest, predictions )<compute_test_metric>
start_time = time.time()
print("Loading data...")
 train = pd.read_csv(".. /input/train.csv" ).fillna(' ')
 test = pd.read_csv('.. /input/test.csv' ).fillna(' ')
 train_text = train['question_text']
test_text = test['question_text']
text_list = pd.concat([train_text, test_text])
 y = train['target'].values
num_train_data = y.shape[0]
print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
print("Spacy NLP...")
 nlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])
 nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)
 word_dict = {}
word_index = 1
lemma_dict = {}
docs = nlp.pipe(text_list, n_threads = 2)
 word_sequences = []
for doc in tqdm(docs):
word_seq = []
for token in doc:
if(token.text not in word_dict)and(token.pos_ is not "PUNCT"):
word_dict[token.text] = word_index
word_index += 1
lemma_dict[token.text] = token.lemma_
if token.pos_ is not "PUNCT":
word_seq.append(word_dict[token.text])
 word_sequences.append(word_seq)
 del docs
gc.collect()
train_word_sequences = word_sequences[:num_train_data]
test_word_sequences = word_sequences[num_train_data:]
del word_sequences
gc.collect()
print("--- %s seconds ---" %(time.time() - start_time))
max_length = 55
embedding_size = 600
learning_rate = 0.001
batch_size = 512
num_epoch = 4
train_word_sequences = pad_sequences(train_word_sequences, maxlen=max_length, padding='post')
 test_word_sequences = pad_sequences(test_word_sequences, maxlen=max_length, padding='post')
 pred_prob = np.zeros(( len(test_word_sequences),), dtype=np.float32)
 start_time = time.time()
print("Loading embedding matrix...")
 embedding_matrix_glove, nb_words = load_glove(word_dict, lemma_dict)
 embedding_matrix_fasttext, nb_words = load_fasttext(word_dict, lemma_dict)
 embedding_matrix = np.concatenate(( embedding_matrix_glove, embedding_matrix_fasttext), axis=1)
 print("--- %s seconds ---" %(time.time() - start_time))
tree = DecisionTreeClassifier(random_state = 11, max_features = "auto", class_weight = "balanced",max_depth = None)
 model_ada = AdaBoostClassifier(base_estimator=tree)
 model_ada = model_ada.fit(xtrain, ytrain)
 predictions = model_ada.predict(xtest)
 print("Accuracy score: ", accuracy_score(ytest, predictions))
print("Precision score: ", precision_score(ytest, predictions))
print("Recall score: ", recall_score(ytest, predictions))
print("F1 score : ", f1_score(predictions, ytest))
confusion_matrix(ytest, predictions )<train_model>
print("Start training...")
 start_time = time.time()
model = build_model(embedding_matrix, nb_words, embedding_size)
 model.summary()
model_gb = GradientBoostingClassifier(criterion='friedman_mse', learning_rate= 0.15,
loss= 'deviance', max_depth= 8, max_features='sqrt',
min_samples_leaf= 0.15714285714285714, min_samples_split= 0.5,
n_estimators= 10, subsample=1.0)
 model_gb = model_gb.fit(xtrain, ytrain)
 predictions = model_gb.predict(xtest)
 print("Accuracy score: ", accuracy_score(ytest, predictions))
print("Precision score: ", precision_score(ytest, predictions))
print("Recall score: ", recall_score(ytest, predictions))
print("F1 score : ", f1_score(predictions, ytest))
confusion_matrix(ytest, predictions )<train_model>
model.fit(train_word_sequences, y, batch_size=batch_size, epochs=num_epoch-1, verbose=2)
 pred_prob += 0.15*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))
model.fit(train_word_sequences, y, batch_size=batch_size, epochs=1, verbose=2)
 pred_prob += 0.35*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))
del model, embedding_matrix_fasttext, embedding_matrix
gc.collect()
K.clear_session()
print("--- %s seconds ---" %(time.time() - start_time))
model = ExtraTreesClassifier(bootstrap=False, criterion='gini', max_depth= None,
max_features= 3, min_samples_leaf= 1, min_samples_split= 10,
n_estimators= 300)
 model.fit(xtrain, ytrain)
 predictions = model.predict(xtest)
 print("Accuracy score: ", accuracy_score(ytest, predictions))
print("Precision score: ", precision_score(ytest, predictions))
print("Recall score: ", recall_score(ytest, predictions))
print("F1 score : ", f1_score(predictions, ytest))
confusion_matrix(ytest, predictions )<save_to_csv>
start_time = time.time()
print("Loading embedding matrix...")
 embedding_matrix_para, nb_words = load_para(word_dict, lemma_dict)
 embedding_matrix = np.concatenate(( embedding_matrix_glove, embedding_matrix_para), axis=1)
 print("--- %s seconds ---" %(time.time() - start_time))
del embedding_matrix_para, embedding_matrix_glove, word_dict, lemma_dict
gc.collect()
y_pred = model_gb.predict(vector_test)
 sample_sub=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 y_pred = np.round(y_pred ).astype(int ).reshape(3263)
 sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':y_pred})
 sub.to_csv('submission_gb.csv',index=False)
 y_pred = model_ada.predict(vector_test)
 sample_sub=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 y_pred = np.round(y_pred ).astype(int ).reshape(3263)
 sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':y_pred})
 sub.to_csv('submission_ada.csv',index=False)
 y_pred = model_xgb.predict(vector_test)
 sample_sub=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 y_pred = np.round(y_pred ).astype(int ).reshape(3263)
 sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':y_pred})
 sub.to_csv('submission_xgb.csv',index=False)
 y_pred = model_lr.predict(vector_test)
 sample_sub=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 y_pred = np.round(y_pred ).astype(int ).reshape(3263)
 sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':y_pred})
 sub.to_csv('submission_lr.csv',index=False)
 y_pred = model.predict(vector_test)
 sample_sub=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 y_pred = np.round(y_pred ).astype(int ).reshape(3263)
 sub=pd.DataFrame({'id':sample_sub['id'].values.tolist() ,'target':y_pred})
 sub.to_csv('submission_ebc.csv',index=False )<import_modules>
start_time = time.time()
print("Start training...")
 model = build_model(embedding_matrix, nb_words, embedding_size)
 del embedding_matrix
gc.collect()
model.fit(train_word_sequences, y, batch_size=batch_size, epochs=num_epoch-1, verbose=2)
 pred_prob += 0.15*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))
model.fit(train_word_sequences, y, batch_size=batch_size, epochs=1, verbose=2)
 pred_prob += 0.35*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))
print("--- %s seconds ---" %(time.time() - start_time))
submission = pd.DataFrame.from_dict({'qid': test['qid']})
 submission['prediction'] =(pred_prob>0.35 ).astype(int)
 submission.to_csv('submission.csv', index=False )