Repository: mozilla/bugbug

Issue:
Iterate bugs only once when training, not twice
Currently we are iterating them once for getting y, and once for getting X. We should just iterate once.


Discussion:

Resolving patch:
diff --git a/bugbug/model.py b/bugbug/model.py
--- a/bugbug/model.py
+++ b/bugbug/model.py
@@ -14,6 +14,7 @@
 
 from bugbug import bugzilla, repository
 from bugbug.nlp import SpacyVectorizer
+from bugbug.utils import split_tuple_iterator
 
 
 class Model:
@@ -65,17 +66,14 @@ def train(self, importance_cutoff=0.15):
         classes, class_names = self.get_labels()
         class_names = sorted(list(class_names), reverse=True)
 
-        # Get items, filtering out those for which we have no labels.
-        def trainable_items_gen():
-            return (
-                item for item in self.items_gen(classes) if self.get_id(item) in classes
-            )
-
-        # Calculate labels.
-        y = np.array([classes[self.get_id(item)] for item in trainable_items_gen()])
+        # Get items and labels, filtering out those for which we have no labels.
+        X_iter, y_iter = split_tuple_iterator(self.items_gen(classes))
 
         # Extract features from the items.
-        X = self.extraction_pipeline.fit_transform(trainable_items_gen())
+        X = self.extraction_pipeline.fit_transform(X_iter)
+
+        # Calculate labels.
+        y = np.array(y_iter)
 
         print(f"X: {X.shape}, y: {y.shape}")
 
@@ -220,28 +218,28 @@ def check(self):
 
 
 class BugModel(Model):
-    def get_id(self, bug):
-        return bug["id"]
-
     def items_gen(self, classes):
-        return (bug for bug in bugzilla.get_bugs())
+        for bug in bugzilla.get_bugs():
+            if bug["id"] not in classes:
+                continue
 
+            yield bug, classes[bug["id"]]
 
-class CommitModel(Model):
-    def get_id(self, commit):
-        return commit["node"]
 
+class CommitModel(Model):
     def items_gen(self, classes):
-        return (commit for commit in repository.get_commits())
+        for commit in repository.get_commits():
+            if commit["node"] not in classes:
+                continue
 
+            yield commit, classes[commit["node"]]
 
-class BugCoupleModel(Model):
-    def get_id(self, bug):
-        return bug[0]["id"], bug[1]["id"]
 
+class BugCoupleModel(Model):
     def items_gen(self, classes):
         bugs = {}
         for bug in bugzilla.get_bugs():
             bugs[bug["id"]] = bug
 
-        return ((bugs[bug_id1], bugs[bug_id2]) for bug_id1, bug_id2 in classes)
+        for (bug_id1, bug_id2), label in classes.items():
+            yield (bugs[bug_id1], bugs[bug_id2]), label
diff --git a/bugbug/utils.py b/bugbug/utils.py
--- a/bugbug/utils.py
+++ b/bugbug/utils.py
@@ -3,6 +3,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import collections
 import os
 
 import numpy as np
@@ -14,6 +15,17 @@
 TASKCLUSTER_DEFAULT_URL = "https://taskcluster.net"
 
 
+def split_tuple_iterator(iterable):
+    q = collections.deque()
+
+    def first_iter():
+        for first, second in iterable:
+            yield first
+            q.append(second)
+
+    return first_iter(), q
+
+
 def numpy_to_dict(array):
     return {name: array[name].squeeze(axis=1) for name in array.dtype.names}
 


Test patch: