Kaggle competition: Quora Question Pairs
for index, row in train_df_se.iterrows() :
if row["selected_text_se"] not in row["text_se"]:
print("Error at = " + str(index), row["selected_text_se"], row["text_se"], )<define_variables>
import pandas as pd
import numpy as np
import gc
counter = 0
for index, row in train_df_se.iterrows() :
if row["selected_text_se"] != row["text_se"]:
counter += 1
print("Sentiment = ",row["sentiment"].upper() + "--" +row["position"])
 print("Real word = ",row["text_se"])
 print("Selected word = ",row["selected_text_se"])
 print("==========================================")
 print("Different word number in train set =", counter )<string_transform>
nrows = 10000000
path = '.. /input/quora-question-pairs/'
train = pd.read_csv(path+"train.csv",nrows=nrows ).astype(str)
 test = pd.read_csv(path+"test.csv",nrows=nrows ).astype(str )
text_se = []
selected_text_se = []
sentiment_se = []
position = []
for index, row in validation_df.iterrows() :
text = " "+" ".join(row['text'].split())
 selected_text = " ".join(row["predicted_text"].split())
 selected_text_start_word = selected_text.split() [0]
selected_text_end_word = selected_text.split() [-1]
selected_text_start_char = text.find(selected_text)
 selected_text_end_char = selected_text_start_char + len(selected_text)
 selected_text_full_start_word = None
selected_text_full_end_word = None
char_counter = 0
for word in row["text"].split() :
char_counter += len(word)+1
if selected_text_start_char < char_counter:
selected_text_full_start_word = word
break
char_counter = 0
for word in row["text"].split() :
char_counter += len(word)+1
if selected_text_end_char <= char_counter:
selected_text_full_end_word = word
break
if row["sentiment"] == "neutral":
position.append("Start-Neutral")
 text_se.append(selected_text_full_start_word)
 selected_text_se.append(selected_text_start_word)
 position.append("End-Neutral")
 text_se.append(selected_text_full_end_word)
 selected_text_se.append(selected_text_end_word)
 sentiment_se.append(row["sentiment"])
 sentiment_se.append(row["sentiment"])
 else:
position.append("Start")
 text_se.append(selected_text_full_start_word)
 selected_text_se.append(selected_text_start_word)
 position.append("End")
 text_se.append(selected_text_full_end_word)
 selected_text_se.append(selected_text_end_word)
 sentiment_se.append(row["sentiment"])
 sentiment_se.append(row["sentiment"])
 validation_df_se = pd.DataFrame(data={"text_se":text_se, "selected_text_se":selected_text_se, "sentiment":sentiment_se, "position":position})
 validation_df_se<compute_test_metric>
from sklearn.metrics import log_loss
for index, row in validation_df_se.iterrows() :
if row["selected_text_se"] not in row["text_se"]:
print("Error at = " + str(index), row["selected_text_se"], row["text_se"], )<string_transform>
def try_divide(x, y, val=0.0):

 if y != 0.0:
val = float(x)/ y
return val
def get_jaccard(seq1, seq2):

 set1, set2 = set(seq1), set(seq2)
 return 1 - try_divide(len(set1 & set2),float(len(set1 | set2)))
 def get_dice(A,B):
A, B = set(A), set(B)
 intersect = len(A.intersection(B))
union = float(len(A)+ len(B))
d = try_divide(2*intersect, union)
 return d
def get_sorensen(seq1, seq2):

 set1, set2 = set(seq1), set(seq2)
 return 1- try_divide(2 * len(set1 & set2),float(len(set1)+ len(set2)))
 def get_count_q1_in_q2(seq1,seq2):
set1, set2 = set(seq1), set(seq2)
 return len(set1 & set2)
 def get_ratio_q1_in_q2(seq1,seq2):
set1, set2 = set(seq1), set(seq2)
 try:
return len(set1 & set2)/float(len(set1))
except:
return 0.0
def get_count_of_question(seq1):
return len(seq1)
 def get_count_of_unique_question(seq1):
set1 = set(seq1)
 return len(set1)
 def get_ratio_of_unique_question(seq1):
set1 = set(seq1)
 try:
return len(set1)/float(len(seq1))
except:
return 0.0
def get_count_of_digit(seq1):
return sum([1.for w in seq1 if w.isdigit() ])
 def get_ratio_of_digit(seq1):
try:
return sum([1.for w in seq1 if w.isdigit() ])/float(len(seq1))
except:
return 0.0
def get_sim_feature(X_batch_1,X_batch_2):
X_jaccard = np.array([ get_jaccard(x1,x2)for x1,x2 in zip(X_batch_1,X_batch_2)] ).reshape(-1,1)
 X_dice = np.array([ get_dice(x1,x2)for x1,x2 in zip(X_batch_1,X_batch_2)] ).reshape(-1,1)
 X_count = np.array([ get_count_q1_in_q2(x1,x2)for x1,x2 in zip(X_batch_1,X_batch_2)] ).reshape(-1,1)
 X_ratio = np.array([ get_ratio_q1_in_q2(x1,x2)for x1,x2 in zip(X_batch_1,X_batch_2)] ).reshape(-1,1)
 X_len1 = np.array([ get_count_of_question(x1)for x1 in X_batch_1] ).reshape(-1,1)
 X_len2 = np.array([ get_count_of_question(x2)for x2 in X_batch_2] ).reshape(-1,1)
 X_len1_unique = np.array([ get_count_of_unique_question(x1)for x1 in X_batch_1] ).reshape(-1,1)
 X_len2_unique = np.array([ get_count_of_unique_question(x2)for x2 in X_batch_2] ).reshape(-1,1)
 X_len_diff = np.abs(X_len2-X_len1)
 X_batch_sim = np.hstack([X_jaccard,X_dice,X_count,X_ratio,X_len1,X_len2,X_len1_unique,X_len2_unique,X_len_diff])
 return X_batch_sim
text_se = []
selected_text_se = []
sentiment_se = []
position = []
for index, row in test_df.iterrows() :
text = " "+" ".join(row['text'].split())
 selected_text = " ".join(row["selected_text"].split())
 selected_text_start_word = selected_text.split() [0]
selected_text_end_word = selected_text.split() [-1]
selected_text_start_char = text.find(selected_text)
 selected_text_end_char = selected_text_start_char + len(selected_text)
 selected_text_full_start_word = None
selected_text_full_end_word = None
char_counter = 0
for word in row["text"].split() :
char_counter += len(word)+1
if selected_text_start_char < char_counter:
selected_text_full_start_word = word
break
char_counter = 0
for word in row["text"].split() :
char_counter += len(word)+1
if selected_text_end_char <= char_counter:
selected_text_full_end_word = word
break
if row["sentiment"] == "neutral":
position.append("Start-Neutral")
 text_se.append(selected_text_full_start_word)
 selected_text_se.append(selected_text_start_word)
 position.append("End-Neutral")
 text_se.append(selected_text_full_end_word)
 selected_text_se.append(selected_text_end_word)
 sentiment_se.append(row["sentiment"])
 sentiment_se.append(row["sentiment"])
 else:
position.append("Start")
 text_se.append(selected_text_full_start_word)
 selected_text_se.append(selected_text_start_word)
 position.append("End")
 text_se.append(selected_text_full_end_word)
 selected_text_se.append(selected_text_end_word)
 sentiment_se.append(row["sentiment"])
 sentiment_se.append(row["sentiment"])
 test_df_se = pd.DataFrame(data={"text_se":text_se, "selected_text_se":selected_text_se, "sentiment":sentiment_se, "position":position})
 test_df_se<compute_test_metric>
X = get_sim_feature(train['question1'].apply(lambda x:x.split(' ')) ,train['question2'].apply(lambda x:x.split(' ')))
 X_test = get_sim_feature(test['question1'].apply(lambda x:x.split(' ')) ,test['question2'].apply(lambda x:x.split(' ')))
 y = train['is_duplicate'].values
scaler = StandardScaler()
X = scaler.fit_transform(X)
 X_test = scaler.transform(X_test)
 print(X.shape,X_test.shape)
for index, row in test_df_se.iterrows() :
if row["selected_text_se"] not in row["text_se"]:
print("Error at = " + str(index), row["selected_text_se"], row["text_se"], )<define_variables>
corpus = []
feats = ['question1','question2']
for f in feats:
train[f] = train[f].astype(str)
 test[f] = test[f].astype(str)
 corpus+=train[f].values.tolist()
ngram_range =(1,2)
 max_features = 50000
min_df = 3
tfidf = TfidfVectorizer(max_features=max_features,ngram_range=ngram_range, min_df=min_df)
 tfidf.fit(corpus)
 X_tfidf_q1 = tfidf.transform(train['question1'])
 X_tfidf_q2 = tfidf.transform(train['question2'])
 X_tfidf_test_q1 = tfidf.transform(test['question1'])
 X_tfidf_test_q2 = tfidf.transform(test['question2'])
 X_tfidf = ssp.hstack([X_tfidf_q1,X_tfidf_q2] ).tocsr()
X_tfidf_test = ssp.hstack([X_tfidf_test_q1,X_tfidf_test_q2] ).tocsr()
del X_tfidf_q1
del X_tfidf_q2
del X_tfidf_test_q1
del X_tfidf_test_q2
gc.collect()
counter = 0
for index, row in test_df_se.iterrows() :
if row["selected_text_se"] != row["text_se"]:
counter += 1
print("Sentiment = ",row["sentiment"].upper() + "--" +row["position"])
 print("Real word = ",row["text_se"])
 print("Selected word = ",row["selected_text_se"])
 print("==========================================")
 print("Different word number in test set =", counter )<import_modules>
svd = TruncatedSVD(100)
 svd.fit(ssp.vstack([X_tfidf,X_tfidf_test]))
X_svd = svd.transform(X_tfidf)
 X_test_svd = svd.transform(X_tfidf_test )
spacy.prefer_gpu() 
<train_model>
X = np.hstack([X,X_svd])
 X_test = np.hstack([X_test,X_test_svd])
def save_model(output_dir, nlp, new_model_name):

 output_dir = f'.. /working/{output_dir}'
if output_dir is not None:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
 nlp.meta["name"] = new_model_name
nlp.to_disk(output_dir)
 print("Saved model to", output_dir )<load_pretrained>
X_train,X_val,y_train,y_val = train_test_split(X,y,train_size=0.8,random_state=1024)
 print(X_train.shape,X_val.shape )
def train(train_data, output_dir, n_iter=20, cont=False):

 ""
if cont and os.path.exists(output_dir):
nlp = spacy.load(output_dir)
 print("Loaded model '%s'" % output_dir)
 else:
nlp = spacy.blank("en")
 print("Created blank 'en' model")
 if "ner" not in nlp.pipe_names:
ner = nlp.create_pipe("ner")
 nlp.add_pipe(ner, last=True)
 else:
ner = nlp.get_pipe("ner")
 for _, annotations in train_data:
for ent in annotations.get("entities"):
ner.add_label(ent[2])
 other_pipes = [pipe for pipe in nlp.pipe_names if pipe != "ner"]
with nlp.disable_pipes(*other_pipes):
if cont and os.path.exists(output_dir):
nlp.resume_training()
else:
nlp.begin_training()
for itn in tqdm(range(n_iter)) :
random.shuffle(train_data)
 batches = minibatch(train_data, size=compounding(4.0, 500.0, 1.001))
losses = {}
for batch in batches:
texts, annotations = zip(*batch)
 nlp.update(texts,
annotations,
drop=0.5,
losses=losses, 
)
 print("Losses", losses)
 save_model(output_dir, nlp, 'st_ner' )<define_variables>
lgb_params = {"boosting": "gbdt", 'learning_rate': 0.05,
"feature_fraction": 0.6, "bagging_freq": 1, "bagging_fraction": 0.8 , 'n_estimators': 100000,
"metric": 'mae', "lambda_l1": 0.1, 'num_leaves': 32, 'min_data_in_leaf': 50, "verbose": 1, "num_threads": 8,
"bagging_seed" : 1024,
"seed": 1024,
'feature_fraction_seed': 1024,
}
clf = lgb.LGBMClassifier(**lgb_params)
 clf.fit(X_train, y_train, eval_set=[(X_train, y_train),(X_val, y_val)], verbose = 100, eval_metric ='logloss',early_stopping_rounds=200)
 y_pred = clf.predict_proba(X_val)[:,1]
eval_score = log_loss(y_val,y_pred)
 print(eval_score)
<train_model><EOS> 
y_pred_test = clf.predict_proba(X_test)[:,1]
submission = pd.DataFrame()
submission['test_id'] = test['test_id']
submission['is_duplicate'] = y_pred_test
submission.to_csv('submission_lgb.csv',index=False)