Upload 5 files

app.py

@@ -0,0 +1,20 @@
+import streamlit as st
+import helper
+import pickle
+import sklearn
+from sklearn.feature_extraction.text import CountVectorizer
+
+model = pickle.load(open('model.pkl', 'rb'))
+st.header("Duplicate Question Pairs")
+
+q1 = st.text_input("Enter question 1")
+q2 = st.text_input("Enter question 2")
+
+if st.button("Find"):
+    query = helper.query_point_creator(q1, q2)
+    result = model.predict(query)[0]
+
+    if result:
+        st.header("Duplicate")
+    else:
+        st.header("Not Duplicate")

cv.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7990f044f2698296a6d944f70f0d10f55296eeb9544011d474748428deb2692c
+size 543885

helper.py

@@ -0,0 +1,266 @@
+import pandas as pd
+import numpy as np
+from bs4 import BeautifulSoup
+import re
+
+import nltk
+from nltk.corpus import stopwords
+nltk.download('stopwords')
+
+import distance
+from fuzzywuzzy import fuzz
+import pickle
+
+import sklearn
+from sklearn.feature_extraction.text import CountVectorizer
+
+cv = pickle.load(open('cv.pkl', 'rb'))
+def preprocess(q):
+    q = str(q).lower().strip()
+
+    q = q.replace('%', ' percent')
+    q = q.replace('$', ' dollar ')
+    q = q.replace('₹', ' rupee ')
+    q = q.replace('€', ' euro ')
+    q = q.replace('@', ' at ')
+
+    contractions = {
+        "ain't": "am not",
+        "aren't": "are not",
+        "can't": "can not",
+        "can't've": "can not have",
+        "'cause": "because",
+        "could've": "could have",
+        "couldn't": "could not",
+        "couldn't've": "could not have",
+        "didn't": "did not",
+        "doesn't": "does not",
+        "don't": "do not",
+        "hadn't": "had not",
+        "hadn't've": "had not have",
+        "hasn't": "has not",
+        "haven't": "have not",
+        "he'd": "he would",
+        "he'd've": "he would have",
+        "he'll": "he will",
+        "he'll've": "he will have",
+        "he's": "he is",
+        "how'd": "how did",
+        "how'd'y": "how do you",
+        "how'll": "how will",
+        "how's": "how is",
+        "i'd": "i would",
+        "i'd've": "i would have",
+        "i'll": "i will",
+        "i'll've": "i will have",
+        "i'm": "i am",
+        "i've": "i have",
+        "isn't": "is not",
+        "it'd": "it would",
+        "it'd've": "it would have",
+        "it'll": "it will",
+        "it'll've": "it will have",
+        "it's": "it is",
+        "let's": "let us",
+        "ma'am": "madam",
+        "mayn't": "may not",
+        "might've": "might have",
+        "mightn't": "might not",
+        "mightn't've": "might not have",
+        "must've": "must have",
+        "mustn't": "must not",
+        "mustn't've": "must not have",
+        "needn't": "need not",
+        "needn't've": "need not have",
+        "o'clock": "of the clock",
+        "oughtn't": "ought not",
+        "oughtn't've": "ought not have",
+        "shan't": "shall not",
+        "sha'n't": "shall not",
+        "shan't've": "shall not have",
+        "she'd": "she would",
+        "she'd've": "she would have",
+        "she'll": "she will",
+        "she'll've": "she will have",
+        "she's": "she is",
+        "should've": "should have",
+        "shouldn't": "should not",
+        "shouldn't've": "should not have",
+        "so've": "so have",
+        "so's": "so as",
+        "that'd": "that would",
+        "that'd've": "that would have",
+        "that's": "that is",
+        "there'd": "there would",
+        "there'd've": "there would have",
+        "there's": "there is",
+        "they'd": "they would",
+        "they'd've": "they would have",
+        "they'll": "they will",
+        "they'll've": "they will have",
+        "they're": "they are",
+        "they've": "they have",
+        "to've": "to have",
+        "wasn't": "was not",
+        "we'd": "we would",
+        "we'd've": "we would have",
+        "we'll": "we will",
+        "we'll've": "we will have",
+        "we're": "we are",
+        "we've": "we have",
+        "weren't": "were not",
+        "what'll": "what will",
+        "what'll've": "what will have",
+        "what're": "what are",
+        "what's": "what is",
+        "what've": "what have",
+        "when's": "when is",
+        "when've": "when have",
+        "where'd": "where did",
+        "where's": "where is",
+        "where've": "where have",
+        "who'll": "who will",
+        "who'll've": "who will have",
+        "who's": "who is",
+        "who've": "who have",
+        "why's": "why is",
+        "why've": "why have",
+        "will've": "will have",
+        "won't": "will not",
+        "won't've": "will not have",
+        "would've": "would have",
+        "wouldn't": "would not",
+        "wouldn't've": "would not have",
+        "y'all": "you all",
+        "y'all'd": "you all would",
+        "y'all'd've": "you all would have",
+        "y'all're": "you all are",
+        "y'all've": "you all have",
+        "you'd": "you would",
+        "you'd've": "you would have",
+        "you'll": "you will",
+        "you'll've": "you will have",
+        "you're": "you are",
+        "you've": "you have"
+    }
+
+    q_decontracted = []
+
+    for word in q.split():
+        if word in contractions:
+            word = contractions[word]
+
+        q_decontracted.append(word)
+
+    q = ' '.join(q_decontracted)
+    q = q.replace("'ve", " have")
+    q = q.replace("n't", " not")
+    q = q.replace("'re", " are")
+    q = q.replace("'ll", " will")
+
+    # Removing HTML tags
+    q = BeautifulSoup(q)
+    q = q.get_text()
+
+    # Remove punctuations
+    pattern = re.compile('\W')
+    q = re.sub(pattern, ' ', q).strip()
+
+    return q
+
+
+def test_common_words(q1, q2):
+  w1 = set(map(lambda words : words.lower().strip(), q1.split(" ")))
+  w2 = set(map(lambda words : words.lower().strip(), q2.split(" ")))
+  return len(w1 & w2)
+
+def test_total_words(q1, q2):
+  w1 = set(map(lambda words : words.strip(), q1.split(" ")))
+  w2 = set(map(lambda words : words.strip(), q2.split(" ")))
+  return (len(w1) + len(w2))
+
+def fetch_test_features(q1, q2):
+  SAFE_DIV = 0.00001
+  STOP_WORDS = stopwords.words('english')
+  token_features = [0.0] * 11
+
+  q1_tokens = q1.split()
+  q2_tokens = q2.split()
+
+  if len(q1_tokens) == 0 or len(q2_tokens) == 0:
+    return token_features
+
+  q1_words = set([word for word in q1_tokens if word not in STOP_WORDS])
+  q2_words = set([word for word in q2_tokens if word not in STOP_WORDS])
+
+  q1_stop = set([word for word in q1_tokens if word in STOP_WORDS])
+  q2_stop = set([word for word in q2_tokens if word in STOP_WORDS])
+
+  common_word_cnt = len(q1_words.intersection(q2_words))
+  common_stop_cnt = len(q1_stop.intersection(q2_stop))
+  common_token_cnt = len(set(q1_tokens).intersection(set(q2_tokens)))
+
+  token_features[0] = round(common_word_cnt / (min(len(q1_words), len(q2_words)) + SAFE_DIV), 2)
+  token_features[1] = round(common_word_cnt / (max(len(q1_words), len(q2_words)) + SAFE_DIV), 2)
+
+  token_features[2] = round(common_stop_cnt / (min(len(q1_stop), len(q2_stop)) + SAFE_DIV), 2)
+  token_features[3] = round(common_stop_cnt / (max(len(q1_stop), len(q2_stop)) + SAFE_DIV), 2)
+
+  token_features[4] = round(common_token_cnt / (min(len(q1_tokens), len(q2_tokens)) + SAFE_DIV), 2)
+  token_features[5] = round(common_token_cnt / (max(len(q1_tokens), len(q2_tokens)) + SAFE_DIV), 2)
+
+  token_features[6] = int(q1_tokens[-1] == q2_tokens[-1])
+  token_features[7] = int(q1_tokens[0] == q2_tokens[0])
+
+  token_features[8] = abs(len(q1_tokens) - len(q2_tokens))
+  token_features[9] = round((len(q1_tokens) - len(q2_tokens)) / 2, 2)
+
+  strs = list(distance.lcsubstrings(q1, q2))
+  token_features[10] = round(len(strs[0]) / (min(len(q1), len(q2)) + 1), 2)
+  return token_features
+
+
+def fetch_test_fuzzy_features(q1, q2):
+    fuzzy_features = [0.0] * 4
+
+    # fuzz_ratio
+    fuzzy_features[0] = fuzz.QRatio(q1, q2)
+
+    # fuzz_partial_ratio
+    fuzzy_features[1] = fuzz.partial_ratio(q1, q2)
+
+    # token_sort_ratio
+    fuzzy_features[2] = fuzz.token_sort_ratio(q1, q2)
+
+    # token_set_ratio
+    fuzzy_features[3] = fuzz.token_set_ratio(q1, q2)
+
+    return fuzzy_features
+
+
+def query_point_creator(q1, q2):
+    input_query = []
+    q1 = preprocess(q1)
+    q2 = preprocess(q2)
+
+    input_query.append(len(q1))
+    input_query.append(len(q2))
+
+    input_query.append(len(q1.split(" ")))
+    input_query.append(len(q2.split(" ")))
+
+    input_query.append(test_common_words(q1, q2))
+    input_query.append(test_total_words(q1, q2))
+
+    input_query.append(round(input_query[4] / input_query[5], 2))
+
+    token_features = fetch_test_features(q1, q2)
+    input_query.extend(token_features)
+
+    fuzzy_features = fetch_test_fuzzy_features(q1, q2)
+    input_query.extend(fuzzy_features)
+
+    q1_arr = cv.transform([q1]).toarray()
+    q2_arr = cv.transform([q2]).toarray()
+
+    return np.hstack((np.array(input_query).reshape(1, 22), q1_arr, q2_arr))

model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e09f34acdafdfb0dbe60ee9c92b82478dfdb3316dc5acfc4268f067985063b5
+size 88637720