Kaggle competition: Quora Question Pairs
<SOS> metric: LogLoss Kaggle data source: quora-question-pairs<feature_engineering>
eng_stopwords = set(stopwords.words('english'))
color = sns.color_palette()
%matplotlib inline
pd.options.mode.chained_assignment = None
y_test[y_test >= 0.5] = 1
y_test[y_test < 0.5] = 0
y_test<data_type_conversions>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )
y_test = y_test.astype(int)
 y_test<prepare_x_and_y>
question_df=pd.DataFrame(pd.concat([train_df['question1'],train_df['question2']]))
question_df.columns=["questions"]
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
y_pseudo_label = np.append(y, y_test, axis = 0)
 list_sentences_train = train["comment_text"].fillna("_na_" ).values
list_sentences_test = test["comment_text"].fillna("_na_" ).values
list_sentences_train_pseudo = np.append(list_sentences_train, list_sentences_test, axis = 0 )<string_transform>
question_df = pd.DataFrame(pd.concat([train_df['question1'], train_df['question2']]))
question_df.columns = ["questions"]
question_df["No_of_Words"] = question_df["questions"].apply(lambda x : len(str(x ).split()))
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train_pseudo))
list_tokenized_train_pseudo = tokenizer.texts_to_sequences(list_sentences_train_pseudo)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_t_pseudo = pad_sequences(list_tokenized_train_pseudo, maxlen=maxlen)
 X_te = pad_sequences(list_tokenized_test, maxlen=maxlen )<compute_test_metric>
def get_unigrams(que):
return [word for word in word_tokenize(que.lower())if word not in eng_stopwords]
def get_common_unigrams(row):
return len(set(row["unigrams_ques1"] ).intersection(set(row["unigrams_ques2"])))
 def get_common_unigram_ratio(row):
return float(row["unigrams_common_count"])/ max(len(set(row["unigrams_ques1"] ).union(set(row["unigrams_ques2"]))),1)
 train_df["unigrams_ques1"] = train_df['question1'].apply(lambda x: get_unigrams(str(x)))
 train_df["unigrams_ques2"] = train_df['question2'].apply(lambda x: get_unigrams(str(x)))
 train_df["unigrams_common_count"] = train_df.apply(lambda row: get_common_unigrams(row),axis=1)
 train_df["unigrams_common_ratio"] = train_df.apply(lambda row: get_common_unigram_ratio(row), axis=1)
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.strip().split())for o in open(EMBEDDING_FILE))<feature_engineering>
def get_bigrams(que):
return [word for word in word_tokenize(que.lower())if word not in eng_stopwords]
def get_common_bigrams(row):
return len(set(row["bigrams_ques1"] ).intersection(set(row["bigrams_ques2"])))
 def get_common_bigram_ratio(row):
return float(row["bigrams_common_count"])/ max(len(set(row["bigrams_ques1"] ).union(set(row["bigrams_ques2"]))),1)
 train_df["bigrams_ques1"] = train_df['unigrams_ques1'].apply(lambda x: get_unigrams(str(x)))
 train_df["bigrams_ques2"] = train_df['unigrams_ques2'].apply(lambda x: get_unigrams(str(x)))
 train_df["bigrams_common_count"] = train_df.apply(lambda row: get_common_unigrams(row),axis=1)
 train_df["bigrams_common_ratio"] = train_df.apply(lambda row: get_common_unigram_ratio(row), axis=1)
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
def feature_extraction(row):
que1 = str(row['question1'])
 que2 = str(row['question2'])
 out_list = []
unigrams_que1 = [word for word in que1.lower().split() if word not in eng_stopwords]
unigrams_que2 = [word for word in que2.lower().split() if word not in eng_stopwords]
common_unigrams_len = len(set(unigrams_que1 ).intersection(set(unigrams_que2)))
 common_unigrams_ratio = float(common_unigrams_len)/ max(len(set(unigrams_que1 ).union(set(unigrams_que2))),1)
 out_list.extend([common_unigrams_len, common_unigrams_ratio])
 bigrams_que1 = [i for i in ngrams(unigrams_que1, 2)]
bigrams_que2 = [i for i in ngrams(unigrams_que2, 2)]
common_bigrams_len = len(set(bigrams_que1 ).intersection(set(bigrams_que2)))
 common_bigrams_ratio = float(common_bigrams_len)/ max(len(set(bigrams_que1 ).union(set(bigrams_que2))),1)
 out_list.extend([common_bigrams_len, common_bigrams_ratio])
 trigrams_que1 = [i for i in ngrams(unigrams_que1, 3)]
trigrams_que2 = [i for i in ngrams(unigrams_que2, 3)]
common_trigrams_len = len(set(trigrams_que1 ).intersection(set(trigrams_que2)))
 common_trigrams_ratio = float(common_trigrams_len)/ max(len(set(trigrams_que1 ).union(set(trigrams_que2))),1)
 out_list.extend([common_trigrams_len, common_trigrams_ratio])
 return out_list
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(LSTM(300, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(50, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0):
params = {}
params["objective"] = "binary:logistic"
params['eval_metric'] = 'logloss'
params["eta"] = 0.02
params["subsample"] = 0.7
params["min_child_weight"] = 1
params["colsample_bytree"] = 0.7
params["max_depth"] = 4
params["silent"] = 1
params["seed"] = seed_val
num_rounds = 300
plst = list(params.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=100, verbose_eval=10)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 loss = 1
if test_y is not None:
loss = log_loss(test_y, pred_test_y)
 return pred_test_y, loss, model
else:
return pred_test_y, loss, model
model.fit(X_t_pseudo, y_pseudo_label, batch_size=32, epochs=2, validation_split=0.1);<save_to_csv>
train_X = np.vstack(np.array(train_df.apply(lambda row: feature_extraction(row), axis=1)))
 test_X = np.vstack(np.array(test_df.apply(lambda row: feature_extraction(row), axis=1)))
 train_y = np.array(train_df["is_duplicate"])
 test_id = np.array(test_df["test_id"] )
y_test_final = model.predict([X_te], batch_size=1024, verbose=1)
 sample_submission = pd.read_csv(f'{path}{comp}sample_submission.csv')
 sample_submission[list_classes] = y_test_final
sample_submission.to_csv('submission.csv', index=False )<set_options>
train_X_dup = train_X[train_y==1]
train_X_non_dup = train_X[train_y==0]
train_X = np.vstack([train_X_non_dup, train_X_dup, train_X_non_dup, train_X_non_dup])
 train_y = np.array([0]*train_X_non_dup.shape[0] + [1]*train_X_dup.shape[0] + [0]*train_X_non_dup.shape[0] + [0]*train_X_non_dup.shape[0])
 del train_X_dup
del train_X_non_dup
print("Mean target rate : ",train_y.mean() )
warnings.filterwarnings("ignore" )<define_variables>
kf = KFold(n_splits=5, shuffle=True, random_state=2016)
 for dev_index, val_index in kf.split(range(train_X.shape[0])) :
dev_X, val_X = train_X[dev_index,:], train_X[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
preds, lloss, model = runXGB(dev_X, dev_y, val_X, val_y)
 break
<string_transform><EOS> 
xgtest = xgb.DMatrix(test_X)
 preds = model.predict(xgtest)
 out_df = pd.DataFrame({"test_id":test_id, "is_duplicate":preds})
 out_df.to_csv("xgb_starter.csv", index=False )