Repository: mozilla/bugbug

Issue:
Add an option not to cleanup URLs for similarity
In the duplicate case, having two equal URL can be a strong indication that two bugs are the same.

See also #709.


Discussion:

Resolving patch:
diff --git a/bugbug/similarity.py b/bugbug/similarity.py
--- a/bugbug/similarity.py
+++ b/bugbug/similarity.py
@@ -34,16 +34,6 @@
 
 REPORTERS_TO_IGNORE = {"intermittent-bug-filer@mozilla.bugs", "wptsync@mozilla.bugs"}
 
-cleanup_functions = [
-    feature_cleanup.responses(),
-    feature_cleanup.hex(),
-    feature_cleanup.dll(),
-    feature_cleanup.fileref(),
-    feature_cleanup.url(),
-    feature_cleanup.synonyms(),
-    feature_cleanup.crash(),
-]
-
 # A map from bug ID to its duplicate IDs
 duplicates = defaultdict(set)
 all_ids = set(
@@ -62,30 +52,37 @@
         duplicates[dupe].add(bug["id"])
 
 
-def get_text(bug):
-    return "{} {}".format(bug["summary"], bug["comments"][0]["text"])
-
-
-def text_preprocess(text, join=False):
-    for func in cleanup_functions:
-        text = func(text)
+class BaseSimilarity:
+    def __init__(self, cleanup_urls=True):
+        self.cleanup_functions = [
+            feature_cleanup.responses(),
+            feature_cleanup.hex(),
+            feature_cleanup.dll(),
+            feature_cleanup.fileref(),
+            feature_cleanup.synonyms(),
+            feature_cleanup.crash(),
+        ]
+        if cleanup_urls:
+            self.cleanup_functions.append(feature_cleanup.url())
 
-    text = re.sub("[^a-zA-Z0-9]", " ", text)
+    def get_text(self, bug):
+        return "{} {}".format(bug["summary"], bug["comments"][0]["text"])
 
-    ps = PorterStemmer()
-    text = [
-        ps.stem(word)
-        for word in text.lower().split()
-        if word not in set(stopwords.words("english")) and len(word) > 1
-    ]
-    if join:
-        return " ".join(word for word in text)
-    return text
+    def text_preprocess(self, text, join=False):
+        for func in self.cleanup_functions:
+            text = func(text)
 
+        text = re.sub("[^a-zA-Z0-9]", " ", text)
 
-class BaseSimilarity:
-    def __init__(self):
-        pass
+        ps = PorterStemmer()
+        text = [
+            ps.stem(word)
+            for word in text.lower().split()
+            if word not in set(stopwords.words("english")) and len(word) > 1
+        ]
+        if join:
+            return " ".join(word for word in text)
+        return text
 
     def evaluation(self):
         total_r = 0
@@ -152,12 +149,13 @@ def evaluation(self):
 
 
 class LSISimilarity(BaseSimilarity):
-    def __init__(self):
+    def __init__(self, cleanup_urls=True):
+        super().__init__(cleanup_urls=cleanup_urls)
         self.corpus = []
 
         for bug in bugzilla.get_bugs():
 
-            textual_features = text_preprocess(get_text(bug))
+            textual_features = self.text_preprocess(self.get_text(bug))
             self.corpus.append([bug["id"], textual_features])
 
         # Assigning unique integer ids to all words
@@ -183,7 +181,7 @@ def __init__(self):
 
     def get_similar_bugs(self, query, k=10):
         query_summary = "{} {}".format(query["summary"], query["comments"][0]["text"])
-        query_summary = text_preprocess(query_summary)
+        query_summary = self.text_preprocess(query_summary)
 
         # Transforming the query to latent 300-D space
         vec_bow = self.dictionary.doc2bow(query_summary)
@@ -198,14 +196,15 @@ def get_similar_bugs(self, query, k=10):
 
 
 class NeighborsSimilarity(BaseSimilarity):
-    def __init__(self, k=10, vectorizer=TfidfVectorizer()):
+    def __init__(self, k=10, vectorizer=TfidfVectorizer(), cleanup_urls=True):
+        super().__init__(cleanup_urls=cleanup_urls)
         self.vectorizer = vectorizer
         self.similarity_calculator = NearestNeighbors(n_neighbors=k)
         text = []
         self.bug_ids = []
 
         for bug in bugzilla.get_bugs():
-            text.append(text_preprocess(get_text(bug), join=True))
+            text.append(self.text_preprocess(self.get_text(bug), join=True))
             self.bug_ids.append(bug["id"])
 
         self.vectorizer.fit(text)
@@ -213,7 +212,7 @@ def __init__(self, k=10, vectorizer=TfidfVectorizer()):
 
     def get_similar_bugs(self, query):
 
-        processed_query = self.vectorizer.transform([get_text(query)])
+        processed_query = self.vectorizer.transform([self.get_text(query)])
         _, indices = self.similarity_calculator.kneighbors(processed_query)
 
         return [
@@ -222,12 +221,13 @@ def get_similar_bugs(self, query):
 
 
 class Word2VecWmdSimilarity(BaseSimilarity):
-    def __init__(self, cut_off=0.2):
+    def __init__(self, cut_off=0.2, cleanup_urls=True):
+        super().__init__(cleanup_urls=cleanup_urls)
         self.corpus = []
         self.bug_ids = []
         self.cut_off = cut_off
         for bug in bugzilla.get_bugs():
-            self.corpus.append(text_preprocess(get_text(bug)))
+            self.corpus.append(self.text_preprocess(self.get_text(bug)))
             self.bug_ids.append(bug["id"])
 
         indexes = list(range(len(self.corpus)))
@@ -289,7 +289,7 @@ def nbow(document):
 
     def get_similar_bugs(self, query):
 
-        words = text_preprocess(get_text(query))
+        words = self.text_preprocess(self.get_text(query))
         words = [word for word in words if word in self.w2vmodel.wv.vocab]
 
         all_distances = np.array(
diff --git a/scripts/evaluate_similarity.py b/scripts/evaluate_similarity.py
--- a/scripts/evaluate_similarity.py
+++ b/scripts/evaluate_similarity.py
@@ -18,19 +18,31 @@ def parse_args(args):
         help="Similarity algorithm to use",
         choices=["lsi", "neighbors_tfidf", "neighbors_tfidf_bigrams", "word2vec_wmd"],
     )
+    parser.add_argument(
+        "--disable-url-cleanup",
+        help="Don't cleanup urls when training the similarity model",
+        dest="cleanup_urls",
+        default=True,
+        action="store_false",
+    )
     return parser.parse_args(args)
 
 
 def main(args):
     if args.algorithm == "lsi":
-        model = LSISimilarity()
+        model_creator = LSISimilarity
     elif args.algorithm == "neighbors_tfidf":
-        model = NeighborsSimilarity()
+        model_creator = NeighborsSimilarity
     elif args.algorithm == "neighbors_tfidf_bigrams":
-        model = NeighborsSimilarity(vectorizer=TfidfVectorizer(ngram_range=(1, 2)))
+
+        def model_creator(**kwargs):
+            kwargs["vectorizer"] = TfidfVectorizer(ngram_range=(1, 2))
+            return NeighborsSimilarity(**kwargs)
+
     elif args.algorithm == "word2vec_wmd":
-        model = Word2VecWmdSimilarity()
+        model_creator = Word2VecWmdSimilarity
 
+    model = model_creator(cleanup_urls=args.cleanup_urls)
     model.evaluation()
 
 


Test patch: