Kaggle competition: Quora Question Pairs
<SOS> metric: LogLoss Kaggle data source: quora-question-pairs<feature_engineering>
import pandas as pd
import numpy as np
import nltk
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
import re
from string import punctuation
for df in [train, test]:
df['device_deviceCategory_channelGrouping'] = df['device.deviceCategory'] + "_" + df['channelGrouping']
df['channelGrouping_browser'] = df['device.browser'] + "_" + df['channelGrouping']
df['channelGrouping_OS'] = df['device.operatingSystem'] + "_" + df['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
df[i + "_" + j] = df[i] + "_" + df[j]
df['content.source'] = df['trafficSource.adContent'].astype(str)+ "_" + df['source.country']
df['medium.source'] = df['trafficSource.medium'] + "_" + df['source.country']<feature_engineering>
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import Dense, Input, LSTM, Embedding, Bidirectional
from tensorflow.keras import layers, utils, callbacks, optimizers, regularizers
from tqdm import tqdm
for feature in ["totals.hits", "totals.pageviews"]:
info = pd.concat([train, test], sort=False ).groupby("fullVisitorId")[feature].mean()
train["usermean_" + feature] = train.fullVisitorId.map(info)
 test["usermean_" + feature] = test.fullVisitorId.map(info)
 for feature in ["visitNumber"]:
info = pd.concat([train, test], sort=False ).groupby("fullVisitorId")[feature].max()
train["usermax_" + feature] = train.fullVisitorId.map(info)
 test["usermax_" + feature] = test.fullVisitorId.map(info )<define_variables>
print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU')) )
excluded = ['date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', 'visitId', 'visitStartTime',
'month', 'day', 'help']
cat_cols = [f for f in train.columns if(train[f].dtype == 'object' and f not in excluded)]
real_cols = [f for f in train.columns if(not f in cat_cols and f not in excluded)]<count_unique_values>
train = pd.read_csv(".. /input/quora-question-pairs/train.csv.zip")
 test = pd.read_csv(".. /input/quora-question-pairs/test.csv" )
train[cat_cols].nunique()<data_type_conversions>
def clean_dataframe_train(train):
stop_words = ['the','a','an','and','but','if','or','because','as','what','which','this','that','these','those','then',
'just','so','than','such','both','through','about','for','is','of','while','during','to','What','Which',
'Is','If','While','This']
def text_to_wordlist(text, remove_stop_words=True, stem_words=False):
text = re.sub(r"[^A-Za-z0-9]", " ", text)
 text = re.sub(r"what's", "", text)
 text = re.sub(r"What's", "", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"can't", "cannot ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"I'm", "I am", text)
 text = re.sub(r" m ", " am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r"60k", " 60000 ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e-mail", "email", text)
 text = re.sub(r"\s{2,}", " ", text)
 text = re.sub(r"quikly", "quickly", text)
 text = re.sub(r" usa ", " America ", text)
 text = re.sub(r" USA ", " America ", text)
 text = re.sub(r" u s ", " America ", text)
 text = re.sub(r" uk ", " England ", text)
 text = re.sub(r" UK ", " England ", text)
 text = re.sub(r"india", "India", text)
 text = re.sub(r"switzerland", "Switzerland", text)
 text = re.sub(r"china", "China", text)
 text = re.sub(r"chinese", "Chinese", text)
 text = re.sub(r"imrovement", "improvement", text)
 text = re.sub(r"intially", "initially", text)
 text = re.sub(r"quora", "Quora", text)
 text = re.sub(r" dms ", "direct messages ", text)
 text = re.sub(r"demonitization", "demonetization", text)
 text = re.sub(r"actived", "active", text)
 text = re.sub(r"kms", " kilometers ", text)
 text = re.sub(r"KMs", " kilometers ", text)
 text = re.sub(r" cs ", " computer science ", text)
 text = re.sub(r" upvotes ", " up votes ", text)
 text = re.sub(r" iPhone ", " phone ", text)
 text = re.sub(r"\0rs ", " rs ", text)
 text = re.sub(r"calender", "calendar", text)
 text = re.sub(r"ios", "operating system", text)
 text = re.sub(r"gps", "GPS", text)
 text = re.sub(r"gst", "GST", text)
 text = re.sub(r"programing", "programming", text)
 text = re.sub(r"bestfriend", "best friend", text)
 text = re.sub(r"dna", "DNA", text)
 text = re.sub(r"III", "3", text)
 text = re.sub(r"the US", "America", text)
 text = re.sub(r"Astrology", "astrology", text)
 text = re.sub(r"Method", "method", text)
 text = re.sub(r"Find", "find", text)
 text = re.sub(r"banglore", "Banglore", text)
 text = re.sub(r" J K ", " JK ", text)
 text = ''.join([c for c in text if c not in punctuation])
 if remove_stop_words:
text = text.split()
text = [w for w in text if not w in stop_words]
text = " ".join(text)
 if stem_words:
text = text.split()
stemmer = SnowballStemmer('english')
 stemmed_words = [stemmer.stem(word)for word in text]
text = " ".join(stemmed_words)
 return(text)
 def process_questions(question_list, questions, question_list_name, dataframe):
for question in questions:
question_list.append(text_to_wordlist(str(question)))
 if len(question_list)% 100000 == 0:
progress = len(question_list)/len(dataframe)* 100
print("{} is {}% complete.".format(question_list_name, round(progress, 1)))
 train_question1 = []
process_questions(train_question1, train.question1, 'train_question1', train)
 train_question2 = []
process_questions(train_question2, train.question2, 'train_question2', train)
 train["question1"] = train_question1
train["question2"] = train_question2
return train
for col in cat_cols:
lbl = LabelEncoder()
lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))
 train[col] = lbl.transform(list(train[col].values.astype('str')))
 test[col] = lbl.transform(list(test[col].values.astype('str')) )<data_type_conversions>
def clean_dataframe_test(test):
stop_words = ['the','a','an','and','but','if','or','because','as','what','which','this','that','these','those','then',
'just','so','than','such','both','through','about','for','is','of','while','during','to','What','Which',
'Is','If','While','This']
def text_to_wordlist(text, remove_stop_words=True, stem_words=False):
text = re.sub(r"[^A-Za-z0-9]", " ", text)
 text = re.sub(r"what's", "", text)
 text = re.sub(r"What's", "", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"can't", "cannot ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"I'm", "I am", text)
 text = re.sub(r" m ", " am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r"60k", " 60000 ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e-mail", "email", text)
 text = re.sub(r"\s{2,}", " ", text)
 text = re.sub(r"quikly", "quickly", text)
 text = re.sub(r" usa ", " America ", text)
 text = re.sub(r" USA ", " America ", text)
 text = re.sub(r" u s ", " America ", text)
 text = re.sub(r" uk ", " England ", text)
 text = re.sub(r" UK ", " England ", text)
 text = re.sub(r"india", "India", text)
 text = re.sub(r"switzerland", "Switzerland", text)
 text = re.sub(r"china", "China", text)
 text = re.sub(r"chinese", "Chinese", text)
 text = re.sub(r"imrovement", "improvement", text)
 text = re.sub(r"intially", "initially", text)
 text = re.sub(r"quora", "Quora", text)
 text = re.sub(r" dms ", "direct messages ", text)
 text = re.sub(r"demonitization", "demonetization", text)
 text = re.sub(r"actived", "active", text)
 text = re.sub(r"kms", " kilometers ", text)
 text = re.sub(r"KMs", " kilometers ", text)
 text = re.sub(r" cs ", " computer science ", text)
 text = re.sub(r" upvotes ", " up votes ", text)
 text = re.sub(r" iPhone ", " phone ", text)
 text = re.sub(r"\0rs ", " rs ", text)
 text = re.sub(r"calender", "calendar", text)
 text = re.sub(r"ios", "operating system", text)
 text = re.sub(r"gps", "GPS", text)
 text = re.sub(r"gst", "GST", text)
 text = re.sub(r"programing", "programming", text)
 text = re.sub(r"bestfriend", "best friend", text)
 text = re.sub(r"dna", "DNA", text)
 text = re.sub(r"III", "3", text)
 text = re.sub(r"the US", "America", text)
 text = re.sub(r"Astrology", "astrology", text)
 text = re.sub(r"Method", "method", text)
 text = re.sub(r"Find", "find", text)
 text = re.sub(r"banglore", "Banglore", text)
 text = re.sub(r" J K ", " JK ", text)
 text = ''.join([c for c in text if c not in punctuation])
 if remove_stop_words:
text = text.split()
text = [w for w in text if not w in stop_words]
text = " ".join(text)
 if stem_words:
text = text.split()
stemmer = SnowballStemmer('english')
 stemmed_words = [stemmer.stem(word)for word in text]
text = " ".join(stemmed_words)
 return(text)
 def process_questions(question_list, questions, question_list_name, dataframe):
for question in questions:
question_list.append(text_to_wordlist(str(question)))
 if len(question_list)% 100000 == 0:
progress = len(question_list)/len(dataframe)* 100
print("{} is {}% complete.".format(question_list_name, round(progress, 1)))
 test_question1 = []
process_questions(test_question1, test.question1, 'test_question1', test)
 test_question2 = []
process_questions(test_question2, test.question2, 'test_question2', test)
 test["question1"] = test_question1
test["question2"] = test_question2
return test
for col in real_cols:
train[col] = train[col].astype(float)
 test[col] = test[col].astype(float )<drop_column>
train = clean_dataframe_train(train)
 test = clean_dataframe_test(test )
for to_del in ["date", "sessionId", "visitId", "day"]:
del train[to_del]
del test[to_del]<define_variables>
MAX_SEQUENCE_LENGTH = 100
EMBEDDING_DIM = 300
excluded = ['date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', 'visitId', 'visitStartTime', "month", "help"]
cat_cols = [f for f in train.columns if(train[f].dtype == 'int64' and f not in excluded)]
real_cols = [f for f in train.columns if(not f in cat_cols and f not in excluded)]<compute_test_metric>
corpus = []
feats = ['question1','question2']
for f in feats:
train[f] = train[f].astype(str)
 test[f] = test[f].astype(str)
 corpus+=train[f].values.tolist()
tokenizer = Tokenizer()
tokenizer.fit_on_texts(corpus)
 X_q1 = tokenizer.texts_to_sequences(train['question1'])
 X_q2 = tokenizer.texts_to_sequences(train['question2'])
 X_test_q1 = tokenizer.texts_to_sequences(test['question1'])
 X_test_q2 = tokenizer.texts_to_sequences(test['question2'])
 X_q1 = pad_sequences(X_q1, maxlen=MAX_SEQUENCE_LENGTH)
 X_q2 = pad_sequences(X_q2, maxlen=MAX_SEQUENCE_LENGTH)
 X_test_q1 = pad_sequences(X_test_q1, maxlen=MAX_SEQUENCE_LENGTH)
 X_test_q2 = pad_sequences(X_test_q2, maxlen=MAX_SEQUENCE_LENGTH)
 y = train['is_duplicate'].values
word_index = tokenizer.word_index
nb_words = len(word_index)+1
def score(data, y):
validation_res = pd.DataFrame(
 {"fullVisitorId": data["fullVisitorId"].values,
"transactionRevenue": data["totals.transactionRevenue"].values,
"predictedRevenue": np.expm1(y)})
 validation_res = validation_res.groupby("fullVisitorId")["transactionRevenue", "predictedRevenue"].sum().reset_index()
return np.sqrt(mean_squared_error(np.log1p(validation_res["transactionRevenue"].values),
np.log1p(validation_res["predictedRevenue"].values)) )<categorify>
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
glove_path = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
embedding_matrix,unknown_words = build_matrix(word_index,glove_path )
class KFoldValidation() :
def __init__(self, data, n_splits=5):
unique_vis = np.array(sorted(data['fullVisitorId'].astype(str ).unique()))
folds = GroupKFold(n_splits)
 ids = np.arange(data.shape[0])
 self.fold_ids = []
for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):
self.fold_ids.append([
ids[data['fullVisitorId'].astype(str ).isin(unique_vis[trn_vis])],
ids[data['fullVisitorId'].astype(str ).isin(unique_vis[val_vis])]
])
 def validate(self, train, test, features, model, name="", prepare_stacking=False,
fit_params={"early_stopping_rounds": 50, "verbose": 100, "eval_metric": "rmse"}):
model.FI = pd.DataFrame(index=features)
 full_score = 0
if prepare_stacking:
test[name] = 0
train[name] = np.NaN
for fold_id,(trn, val)in enumerate(self.fold_ids):
devel = train[features].iloc[trn]
y_devel = np.log1p(train["totals.transactionRevenue"].iloc[trn])
 valid = train[features].iloc[val]
y_valid = np.log1p(train["totals.transactionRevenue"].iloc[val])
 print("Fold ", fold_id, ":")
 model.fit(devel, y_devel, eval_set=[(valid, y_valid)], **fit_params)
 if len(model.feature_importances_)== len(features):
model.FI['fold' + str(fold_id)] = model.feature_importances_ / model.feature_importances_.sum()
predictions = model.predict(valid)
 predictions[predictions < 0] = 0
print("Fold ", fold_id, " error: ", mean_squared_error(y_valid, predictions)**0.5)
 fold_score = score(train.iloc[val], predictions)
 full_score += fold_score / len(self.fold_ids)
 print("Fold ", fold_id, " score: ", fold_score)
 if prepare_stacking:
train[name].iloc[val] = predictions
test_predictions = model.predict(test[features])
 test_predictions[test_predictions < 0] = 0
test[name] += test_predictions / len(self.fold_ids)
 print("Final score: ", full_score)
 return full_score<choose_model_class>
X_train_q1,X_val_q1,X_train_q2,X_val_q2,y_train,y_val = train_test_split(X_q1,X_q2,y,train_size=0.8,random_state=1024)
 print(X_train_q1.shape,X_val_q1.shape)
 X_train = [X_train_q1,X_train_q2]
X_val = [X_val_q1,X_val_q2]
X_test = [X_test_q1,X_test_q2]
Kfolder = KFoldValidation(train )<choose_model_class>
input_q1 = Input(shape = X_train[0].shape[1])
 input_q2 = Input(shape = X_train[0].shape[1])
 embedding_layer = Embedding(nb_words,
EMBEDDING_DIM,
input_length = X_train[0].shape[1],
weights = [embedding_matrix],
trainable=False)
 embedded_sequences_q1 = embedding_layer(input_q1)
 embedded_sequences_q2 = embedding_layer(input_q2)
 bilstm_layer = Bidirectional(LSTM(64, return_sequences=False))
x1 = bilstm_layer(embedded_sequences_q1)
 x2 = bilstm_layer(embedded_sequences_q2)
 distance = layers.Concatenate()([x1, x2])
 x = Dense(64, activation="relu" )(distance)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[input_q1, input_q2], outputs=preds)
 model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
 model.summary()
utils.plot_model(model, show_shapes=True, expand_nested=True )
lgbmodel = lgb.LGBMRegressor(n_estimators=1000, objective="regression", metric="rmse", num_leaves=31, min_child_samples=100,
learning_rate=0.03, bagging_fraction=0.7, feature_fraction=0.5, bagging_frequency=5,
bagging_seed=2019, subsample=.9, colsample_bytree=.9, use_best_model=True )<train_model>
es = callbacks.EarlyStopping(
 monitor='val_loss', patience=5, verbose=1, restore_best_weights=True 
)
 rlp = callbacks.ReduceLROnPlateau(
 monitor='val_loss', factor=0.5, patience=2, min_lr=1e-10, mode='min', verbose=1 
)
 history = model.fit(
 X_train, y_train,
validation_data=(X_val, y_val),
batch_size=32,
epochs=100,
callbacks=[es, rlp]
)
Kfolder.validate(train, test, real_cols + cat_cols, lgbmodel, "lgbpred", prepare_stacking=True )<create_dataframe>
y_pred_test = model.predict(X_test)
 submission = pd.DataFrame()
submission['test_id'] = test['test_id']
submission['is_duplicate'] = y_pred_test
submission.to_csv('submission.csv',index=False )