internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: mozilla/bugbug

Issue:
Create an option in bug model for analyzing old bugs
Details are here: https://github.com/mozilla/bugbug/issues/244#issuecomment-480284787 and https://github.com/mozilla/bugbug/pull/247#issuecomment-480756156



Discussion:
May I clarify, should this option be added similar to `lemmatization` option here (https://github.com/mozilla/bugbug/blob/2d5a646784eb18b645dbfaae6a5e7401bf1e355a/bugbug/models/bug.py#L19) as well as to args parameters in `run.py`?  And then if true, severity feature will be added to feature_extractors in `bug.py`.
Yes, exactly!

Resolving patch:
diff --git a/bugbug/models/bug.py b/bugbug/models/bug.py
--- a/bugbug/models/bug.py
+++ b/bugbug/models/bug.py
@@ -16,7 +16,7 @@
 
 
 class BugModel(Model):
-    def __init__(self, lemmatization=False):
+    def __init__(self, lemmatization=False, historical=False):
         Model.__init__(self, lemmatization)
 
         self.sampler = BorderlineSMOTE(random_state=0)
@@ -43,6 +43,9 @@ def __init__(self, lemmatization=False):
             bug_features.component(),
         ]
 
+        if historical:
+            feature_extractors.append(bug_features.had_severity_enhancement())
+
         cleanup_functions = [
             bug_features.cleanup_url,
             bug_features.cleanup_fileref,
diff --git a/run.py b/run.py
--- a/run.py
+++ b/run.py
@@ -50,6 +50,7 @@
         action="store_true",
     )
     parser.add_argument("--token", help="Bugzilla token", action="store")
+    parser.add_argument("--historical", help="Analyze historical bugs", action="store_true")
     args = parser.parse_args()
 
     model_file_name = "{}{}model".format(
@@ -101,7 +102,10 @@
     if args.train:
         db.download()
 
-        model = model_class(args.lemmatization)
+        if args.historical:
+            model = model_class(args.lemmatization, args.historical)
+        else:
+            model = model_class(args.lemmatization)
         model.train()
     else:
         model = model_class.load(model_file_name)


Test patch:
Repository: mozilla/bugbug

Issue:
Make Keras/Tensorflow dependencies optional
See also #307.


Discussion:
No image is using these deps yet.

Resolving patch:
diff --git a/bugbug/models/component_nn.py b/bugbug/models/component_nn.py
--- a/bugbug/models/component_nn.py
+++ b/bugbug/models/component_nn.py
@@ -3,18 +3,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from keras import Input, layers
-from keras.layers import (
-    GRU,
-    Bidirectional,
-    Dense,
-    Dropout,
-    Embedding,
-    Flatten,
-    GlobalMaxPooling1D,
-    SpatialDropout1D,
-)
-from keras.models import Model as KerasModel
 from sklearn.ensemble import VotingClassifier
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.pipeline import Pipeline, make_pipeline
@@ -29,10 +17,29 @@
     StructuredColumnTransformer,
 )
 
+OPT_MSG_MISSING = (
+    "Optional dependencies are missing, install them with: pip install bugbug[nn]\n"
+)
+
+try:
+    from keras import Input, layers
+    from keras.layers import (
+        GRU,
+        Bidirectional,
+        Dense,
+        Dropout,
+        Embedding,
+        Flatten,
+        GlobalMaxPooling1D,
+        SpatialDropout1D,
+    )
+    from keras.models import Model as KerasModel
+except ImportError:
+    raise ImportError(OPT_MSG_MISSING)
+
 
 class ComponentNNClassifier(KerasClassifier):
     def __init__(self, **kwargs):
-
         # (epochs, batch_size) combinations
         fit_params = [(2, 256), (2, 512), (1, 1024)]
         super().__init__(fit_params=fit_params)
diff --git a/bugbug/nn.py b/bugbug/nn.py
--- a/bugbug/nn.py
+++ b/bugbug/nn.py
@@ -3,13 +3,23 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from keras.preprocessing.sequence import pad_sequences
-from keras.preprocessing.text import Tokenizer
-from keras.utils import to_categorical
 from sklearn.base import BaseEstimator, ClassifierMixin, TransformerMixin
 
 from bugbug.utils import numpy_to_dict
 
+OPT_MSG_MISSING = (
+    "Optional dependencies are missing, install them with: pip install bugbug[nn]\n"
+)
+
+try:
+    from keras.preprocessing.sequence import pad_sequences
+    from keras.preprocessing.text import Tokenizer
+    from keras.utils import to_categorical
+
+    HAS_OPTIONAL_DEPENDENCIES = True
+except ImportError:
+    raise ImportError(OPT_MSG_MISSING)
+
 
 class KerasTextToSequences(BaseEstimator, TransformerMixin):
     def __init__(self, maxlen, vocab_size):
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -38,7 +38,7 @@ def read_requirements(file_):
     version = f.read().strip()
 
 # Read the extra requirements
-extras = ["nlp"]
+extras = ["nlp", "nn"]
 
 extras_require = {}
 


Test patch:
Repository: mozilla/bugbug

Issue:
Figure out what to do with http_service on CI
We have two options:
- build the http_service with fake models and don't push it on CI. Build it with real models and push it after training;
- build the http_service without models and let it download models at runtime.


Discussion:
We are currently trying to update the index routes and we hit a chicken-and-egg issue. In order to build a functioning http_service Docker image, we need the models that we download from taskcluster. The issue is that the models are not produced until the data pipeline runs and it runs with the new code only after the release.

Resolving patch:
diff --git a/http_service/check_models.py b/http_service/check_models.py
--- a/http_service/check_models.py
+++ b/http_service/check_models.py
@@ -36,6 +36,13 @@ def check_models():
 
 
 if __name__ == "__main__":
+
+    should_check_models = os.environ.get("CHECK_MODELS", "1")
+
+    if should_check_models == "0":
+        print("Skipping checking models as instructed by env var $CHECK_MODELS")
+        sys.exit(0)
+
     try:
         check_models()
     except Exception:


Test patch:
Repository: mozilla/bugbug

Issue:
Not all training tasks need commits DB
Indeed I think none of the ones we currently run as part of the data pipeline need the commits.
We should:
- Make the trainer script only download the DBs which are necessary;
- Remove the dependency on the commit retrieval task in the data-pipeline.yml.


Discussion:

Resolving patch:
diff --git a/scripts/trainer.py b/scripts/trainer.py
--- a/scripts/trainer.py
+++ b/scripts/trainer.py
@@ -73,13 +73,6 @@ def go(self, model):
         logger.info("Decompressing bugs database")
         self.decompress_file("data/bugs.json")
 
-        # Commits.json
-        logger.info("Downloading commits database")
-        commits_url = BASE_URL.format("commits")
-        urlretrieve(f"{commits_url}/commits.json.xz", "data/commits.json.xz")
-        logger.info("Decompressing commits database")
-        self.decompress_file("data/commits.json")
-
         if model == "defect":
             # Train classifier for defect-vs-enhancement-vs-task.
             self.train_defect_enhancement_task()


Test patch:
Repository: mozilla/bugbug

Issue:
Use codespell in precommit hook



Discussion:

Resolving patch:
diff --git a/run.py b/run.py
--- a/run.py
+++ b/run.py
@@ -73,7 +73,7 @@
         elif args.classifier == "nn":
             model_class_name = "component_nn"
         else:
-            raise ValueError(f"Unkown value {args.classifier}")
+            raise ValueError(f"Unknown value {args.classifier}")
 
     model_class = get_model_class(model_class_name)
 


Test patch:
Repository: mozilla/bugbug

Issue:
Use rs_parsepatch to parse patches
It should be faster.


Discussion:

Resolving patch:
diff --git a/bugbug/repository.py b/bugbug/repository.py
--- a/bugbug/repository.py
+++ b/bugbug/repository.py
@@ -16,8 +16,8 @@
 
 import hglib
 import requests
+import rs_parsepatch
 from dateutil.relativedelta import relativedelta
-from parsepatch.patch import Patch
 from tqdm import tqdm
 
 from bugbug import db
@@ -174,21 +174,24 @@ def _transform(commit):
     sizes = []
 
     patch = HG.export(revs=[commit.node.encode("ascii")], git=True)
-    patch_data = Patch.parse_patch(
-        patch.decode("utf-8", "ignore"), skip_comments=False, add_lines_for_new=True
-    )
-    for path, stats in patch_data.items():
-        if "added" not in stats:
-            # Must be a binary file
+    patch_data = rs_parsepatch.get_counts(patch)
+    components = set()
+    for stats in patch_data:
+        if stats["binary"]:
             obj["types"].add("binary")
             continue
 
+        path = stats["filename"]
+        component = path_to_component.get(path)
+        if component:
+            components.add(component)
+
         if is_test(path):
-            obj["test_added"] += len(stats["added"]) + len(stats["touched"])
-            obj["test_deleted"] += len(stats["deleted"]) + len(stats["touched"])
+            obj["test_added"] += stats["added_lines"]
+            obj["test_deleted"] += stats["deleted_lines"]
         else:
-            obj["added"] += len(stats["added"]) + len(stats["touched"])
-            obj["deleted"] += len(stats["deleted"]) + len(stats["touched"])
+            obj["added"] += stats["added_lines"]
+            obj["deleted"] += stats["deleted_lines"]
 
         ext = os.path.splitext(path)[1]
         if ext in [".js", ".jsm"]:
@@ -216,15 +219,13 @@ def _transform(commit):
             type_ = ext
         obj["types"].add(type_)
 
-        try:
-            after = HG.cat([path.encode("utf-8")], rev=commit.node.encode("ascii"))
-        except hglib.error.CommandError as e:
-            if b"no such file in rev" in e.err:
-                after = b""
-            else:
-                raise
-
-        sizes.append(after.count(b"\n"))
+        if not stats["deleted"]:
+            try:
+                after = HG.cat([path.encode("utf-8")], rev=commit.node.encode("ascii"))
+                sizes.append(after.count(b"\n"))
+            except hglib.error.CommandError as e:
+                if b"no such file in rev" not in e.err:
+                    raise
 
     obj["total_file_size"] = sum(sizes)
     obj["average_file_size"] = (
@@ -238,13 +239,7 @@ def _transform(commit):
     # Covert to a list, as a set is not JSON-serializable.
     obj["types"] = list(obj["types"])
 
-    obj["components"] = list(
-        set(
-            path_to_component[path]
-            for path in patch_data.keys()
-            if path_to_component.get(path)
-        )
-    )
+    obj["components"] = list(components)
 
     return obj
 


Test patch:
Repository: mozilla/bugbug

Issue:
Add "presence of image" in attachments as a feature
When users file a bug uploading an image, it's very likely they want to show that there is something wrong with Firefox.
So, the presence of an image between the attachments can be an indication of the bug being a defect.


Discussion:

Resolving patch:
diff --git a/bugbug/bug_features.py b/bugbug/bug_features.py
--- a/bugbug/bug_features.py
+++ b/bugbug/bug_features.py
@@ -258,6 +258,22 @@ def __call__(self, bug, **kwargs):
         )
 
 
+class has_image_attachment_at_bug_creation(object):
+    def __call__(self, bug, **kwargs):
+        return any(
+            "image" in attachment["content_type"]
+            and attachment["creation_time"] == bug["creation_time"]
+            for attachment in bug["attachments"]
+        )
+
+
+class has_image_attachment(object):
+    def __call__(self, bug, **kwargs):
+        return any(
+            "image" in attachment["content_type"] for attachment in bug["attachments"]
+        )
+
+
 class commit_added(object):
     def __call__(self, bug, **kwargs):
         return sum(


Test patch:
Repository: mozilla/bugbug

Issue:
Add "seniority" feature to commit data
Right now we are calculating the experience in terms of number of commits. We should also add a "seniority" feature, which would be the time since the first commit in the repo.


Discussion:
I am new to open source and I would like to contribute. can I take up this issue?
Sure! Any issue which is not assigned is free to work on (see the CONTRIBUTING file with the rules).

The implementation will have to be in bugbug/repository.py (please follow the README to make sure you can run it).
The script is pretty slow, so I suggest limiting the number of commits to analyze. You can simply do that by changing https://github.com/mozilla/bugbug/blob/3c91af9c60f6b94349975a56017a2bc109b28b6a/bugbug/repository.py#L547 to a smaller time span (e.g. months=1).

Resolving patch:
diff --git a/bugbug/repository.py b/bugbug/repository.py
--- a/bugbug/repository.py
+++ b/bugbug/repository.py
@@ -55,6 +55,8 @@
     EXPERIENCE_TIMESPAN_TEXT: defaultdict(lambda: defaultdict(int)),
 }
 
+seniority = defaultdict(int)
+
 # This is only a temporary hack: Should be removed after the template issue with reviewers (https://bugzilla.mozilla.org/show_bug.cgi?id=1528938)
 # gets fixed. Most of this code is copied from https://github.com/mozilla/version-control-tools/blob/2c2812d4a41b690203672a183b1dd85ca8b39e01/pylib/mozautomation/mozautomation/commitparser.py#L129
 def get_reviewers(commit_description, flag_re=None):
@@ -175,6 +177,8 @@ def _transform(commit):
             commit.node
         ]
 
+    obj["seniority"] = seniority[commit.node]
+
     sizes = []
 
     patch = HG.export(revs=[commit.node.encode("ascii")], git=True)
@@ -331,6 +335,17 @@ def calculate_experiences(commits):
     print(f"Analyzing experiences from {len(commits)} commits...")
 
     global experiences_by_commit
+    global seniority
+
+    first_commit_time = defaultdict(int)
+
+    for commit in commits:
+        if commit.author not in first_commit_time:
+            first_commit_time[commit.author] = commit.pushdate
+            seniority[commit.node] = 0
+        else:
+            time_lapse = commit.pushdate - first_commit_time[commit.author]
+            seniority[commit.node] = time_lapse.days
 
     first_pushdate = commits[0].pushdate
 


Test patch:
Repository: mozilla/bugbug

Issue:
defect_enhancement_task Taskcluster task is missing the artifact
From the task () logs:
```
[taskcluster 2019-05-21 08:37:51.436Z] === Task Finished ===
[taskcluster 2019-05-21 08:37:51.519Z] Artifact "public/defectenhancementtaskmodel.xz" not found at "/defectenhancementtaskmodel.xz"
[taskcluster 2019-05-21 08:37:51.927Z] Successful task run with exit code: 0 completed in 471.275 seconds
```


Discussion:

Resolving patch:
diff --git a/scripts/trainer.py b/scripts/trainer.py
--- a/scripts/trainer.py
+++ b/scripts/trainer.py
@@ -43,9 +43,14 @@ def go(self, model_name):
         model = model_class()
         model.train()
 
+        logger.info(f"Training done")
+
         model_file_name = f"{model_name}model"
+        assert os.path.exists(model_file_name)
         self.compress_file(model_file_name)
 
+        logger.info(f"Model compressed")
+
 
 def main():
     description = "Train the models"


Test patch:
Repository: mozilla/bugbug

Issue:
Don't try to find inconsistencies in all bugs multiple times
We are re-downloading inconsistent bugs at https://github.com/mozilla/bugbug/blob/cc9c5e8c5d69e2dcabf35741a3f1d9b9bb797b39/scripts/bug_retriever.py#L61.

To find them, we are iterating through all bugs.

It makes sense to iterate through all bugs the first time, but the second time we should only iterate through the bugs we found inconsistent the first time (indeed, we can be sure that all the other bugs were not inconsistent).

To fix this, we simply need to modify https://github.com/mozilla/bugbug/blob/cc9c5e8c5d69e2dcabf35741a3f1d9b9bb797b39/bugbug/bug_snapshot.py#L755 adding a "bugs" parameter, and call it in the right way in bug_retriever.py (the first time bugs=bugzilla.get_bugs(), the second and following times bugs=previously_inconsistent_bugs).


Discussion:
@marco-c Can I work on this issue ?
Sure! See the CONTRIBUTING.md file, any issue which is not linked to a PR and not assigned is free for everyone to work on.

Resolving patch:
diff --git a/bugbug/bug_snapshot.py b/bugbug/bug_snapshot.py
--- a/bugbug/bug_snapshot.py
+++ b/bugbug/bug_snapshot.py
@@ -752,16 +752,16 @@ def assert_or_log(msg):
     return bug
 
 
-def get_inconsistencies():
+def get_inconsistencies(bugs):
     inconsistencies = []
 
-    for bug in bugzilla.get_bugs():
+    for bug in bugs:
         try:
             rollback(bug, do_assert=True)
         except Exception as e:
             print(bug["id"])
             print(e)
-            inconsistencies.append(bug["id"])
+            inconsistencies.append(bug)
 
     return inconsistencies
 
diff --git a/scripts/bug_retriever.py b/scripts/bug_retriever.py
--- a/scripts/bug_retriever.py
+++ b/scripts/bug_retriever.py
@@ -59,16 +59,21 @@ def retrieve_bugs(self):
         bugzilla.download_bugs(timespan_ids + labelled_bug_ids)
 
         # Try to re-download inconsistent bugs, up to three times.
+        inconsistent_bugs = bugzilla.get_bugs()
         for i in range(3):
-            bug_ids = set(bug_snapshot.get_inconsistencies())
-            if len(bug_ids) == 0:
+            # We look for inconsistencies in all bugs first, then, on following passes,
+            # we only look for inconsistencies in bugs that were found to be inconsistent in the first pass
+            inconsistent_bugs = bug_snapshot.get_inconsistencies(inconsistent_bugs)
+            inconsistent_bug_ids = set(bug["id"] for bug in inconsistent_bugs)
+
+            if len(inconsistent_bug_ids) == 0:
                 break
 
             logger.info(
-                f"Re-downloading {len(bug_ids)} bugs, as they were inconsistent"
+                f"Re-downloading {len(inconsistent_bug_ids)} bugs, as they were inconsistent"
             )
-            bugzilla.delete_bugs(lambda bug: bug["id"] in bug_ids)
-            bugzilla.download_bugs(bug_ids)
+            bugzilla.delete_bugs(lambda bug: bug["id"] in inconsistent_bug_ids)
+            bugzilla.download_bugs(inconsistent_bug_ids)
 
         self.compress_file("data/bugs.json")
 


Test patch:
Repository: mozilla/bugbug

Issue:
Use new 'everchanged' operator instead of changedafter 1970
Depends on https://bugzilla.mozilla.org/show_bug.cgi?id=1546624.


Discussion:

Resolving patch:
diff --git a/scripts/get_type_labels.py b/scripts/get_type_labels.py
--- a/scripts/get_type_labels.py
+++ b/scripts/get_type_labels.py
@@ -27,8 +27,7 @@ def main(args):
         "order": "bug_id",
         "j_top": "OR",
         "f1": "bug_type",
-        "o1": "changedafter",
-        "v1": "1970-01-01",
+        "o1": "everchanged",
         "f2": "OP",
         "f3": "bug_type",
         "o3": "anyexact",


Test patch:
Repository: mozilla/bugbug

Issue:
Add tag in the hook images
Currently we use mozilla/bugbug-spawn-pipeline with no tag.


Discussion:

Resolving patch:
diff --git a/infra/set_hook_env_var.py b/infra/set_hook_version.py
similarity index 62%
rename from infra/set_hook_env_var.py
rename to infra/set_hook_version.py
--- a/infra/set_hook_env_var.py
+++ b/infra/set_hook_version.py
@@ -11,16 +11,10 @@
 def main(raw_args):
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "env_name",
-        metavar="env-name",
+        "version",
+        metavar="version",
         type=str,
-        help="The name of the environment variable to set",
-    )
-    parser.add_argument(
-        "env_value",
-        metavar="env-value",
-        type=str,
-        help="The value of the environment variable to set",
+        help="The version to set in the hook definition",
     )
     parser.add_argument(
         "hook_file",
@@ -34,9 +28,17 @@ def main(raw_args):
     with open(args.hook_file, "r") as hook_file:
         hook_data = json.load(hook_file)
 
-    # Insert or replace the environment variable
-    hook_env = hook_data["task"]["payload"]["env"]
-    hook_env[args.env_name] = args.env_value
+    task_payload = hook_data["task"]["payload"]
+
+    task_image = task_payload.get("image")
+
+    # 1) Insert or replace the environment variable
+    hook_env = task_payload["env"]
+    hook_env["TAG"] = args.version
+
+    # 2) Set the version for the hook docker image
+    if task_image and task_image.split(":", 1)[0] == "mozilla/bugbug-spawn-pipeline":
+        task_payload["image"] = f"mozilla/bugbug-spawn-pipeline:{args.version}"
 
     with open(args.hook_file, "w") as hook_file:
         json.dump(


Test patch:
Repository: mozilla/bugbug

Issue:
Add 'files' to regressor features
I've removed it in 66c496bb25b69ca4b42333ee365755f49faf62fd so that I could enable feature importance for the model.
In order to add it back, we need to implement something similar to TfIdfVectorizer's min_df parameter and only consider files that are most often modified.


Discussion:

Resolving patch:
diff --git a/bugbug/commit_features.py b/bugbug/commit_features.py
--- a/bugbug/commit_features.py
+++ b/bugbug/commit_features.py
@@ -3,6 +3,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from collections import defaultdict
+
 import pandas as pd
 from sklearn.base import BaseEstimator, TransformerMixin
 
@@ -178,8 +180,23 @@ def __call__(self, commit, **kwargs):
 
 
 class files(object):
+    def __init__(self, min_freq=0.00003):
+        self.min_freq = min_freq
+
+    def fit(self, commits):
+        self.count = defaultdict(int)
+
+        for commit in commits:
+            for f in commit["files"]:
+                self.count[f] += 1
+        self.total_files = sum(self.count.values())
+
     def __call__(self, commit, **kwargs):
-        return commit["files"]
+        return [
+            f
+            for f in commit["files"]
+            if (self.count[f] / self.total_files) > self.min_freq
+        ]
 
 
 class file_touched_prev(object):
@@ -202,6 +219,10 @@ def __init__(self, feature_extractors, cleanup_functions):
         self.cleanup_functions = cleanup_functions
 
     def fit(self, x, y=None):
+        for feature in self.feature_extractors:
+            if hasattr(feature, "fit"):
+                feature.fit(x)
+
         return self
 
     def transform(self, commits):
diff --git a/bugbug/models/regressor.py b/bugbug/models/regressor.py
--- a/bugbug/models/regressor.py
+++ b/bugbug/models/regressor.py
@@ -36,6 +36,7 @@ def __init__(self, lemmatization=False):
             commit_features.directory_touched_prev(),
             commit_features.file_touched_prev(),
             commit_features.types(),
+            commit_features.files(),
             commit_features.components(),
             commit_features.components_modified_num(),
             commit_features.directories(),


Test patch: