internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: mozilla/bugbug

Issue:
Add an option not to cleanup URLs for the Duplicate model
In the duplicate case, having two equal URL can be a strong indication that two bugs are the same.


Discussion:

Resolving patch:
diff --git a/bugbug/models/duplicate.py b/bugbug/models/duplicate.py
--- a/bugbug/models/duplicate.py
+++ b/bugbug/models/duplicate.py
@@ -28,7 +28,7 @@ def fit(self, X, y):
 
 
 class DuplicateModel(BugCoupleModel):
-    def __init__(self, training_size=14000, lemmatization=False):
+    def __init__(self, training_size=14000, lemmatization=False, cleanup_urls=True):
         self.num_duplicates = training_size // 2
         self.num_nondups_nondups = self.num_dup_nondups = training_size // 4
 
@@ -43,11 +43,13 @@ def __init__(self, training_size=14000, lemmatization=False):
             feature_cleanup.hex(),
             feature_cleanup.dll(),
             feature_cleanup.fileref(),
-            feature_cleanup.url(),
             feature_cleanup.synonyms(),
             feature_cleanup.crash(),
         ]
 
+        if cleanup_urls:
+            cleanup_functions.append(feature_cleanup.url())
+
         self.extraction_pipeline = Pipeline(
             [
                 (
diff --git a/run.py b/run.py
--- a/run.py
+++ b/run.py
@@ -29,6 +29,13 @@ def parse_args(args):
         type=int,
         help="The size of the training set for the duplicate model",
     )
+    parser.add_argument(
+        "--disable-url-cleanup",
+        help="Don't cleanup urls when training the duplicate model",
+        dest="cleanup_urls",
+        default=True,
+        action="store_false",
+    )
     parser.add_argument("--train", help="Perform training", action="store_true")
     parser.add_argument(
         "--goal", help="Goal of the classifier", choices=MODELS.keys(), default="defect"
@@ -84,7 +91,9 @@ def main(args):
         if args.goal in historical_supported_tasks:
             model = model_class(args.lemmatization, args.historical)
         elif args.goal == "duplicate":
-            model = model_class(args.training_set_size, args.lemmatization)
+            model = model_class(
+                args.training_set_size, args.lemmatization, args.cleanup_urls
+            )
         else:
             model = model_class(args.lemmatization)
         model.train()


Test patch:
Repository: mozilla/bugbug

Issue:
In the trainer script, use db.download instead of implementing download manually
script/trainer.py is currently implementing DB download manually, it should instead use db.download (similarly to scripts/commit_classifier.py).


Discussion:
Would it make sense to just use bug_retriever.retriever?
No, bug_retriever is the script which is generating the DB, we don't want to regenerate it again on training.

Resolving patch:
diff --git a/scripts/trainer.py b/scripts/trainer.py
--- a/scripts/trainer.py
+++ b/scripts/trainer.py
@@ -4,44 +4,24 @@
 import json
 import os
 from logging import INFO, basicConfig, getLogger
-from urllib.request import urlretrieve
 
 import zstandard
 
-from bugbug import get_bugbug_version, model
+from bugbug import bugzilla, db, model, repository
 from bugbug.models import get_model_class
 from bugbug.utils import CustomJsonEncoder
 
 basicConfig(level=INFO)
 logger = getLogger(__name__)
 
-BASE_URL = "https://index.taskcluster.net/v1/task/project.relman.bugbug.data_{}.{}/artifacts/public"
-
 
 class Trainer(object):
-    def decompress_file(self, path):
-        dctx = zstandard.ZstdDecompressor()
-        with open(f"{path}.zst", "rb") as input_f:
-            with open(path, "wb") as output_f:
-                dctx.copy_stream(input_f, output_f)
-        assert os.path.exists(path), "Decompressed file exists"
-
     def compress_file(self, path):
         cctx = zstandard.ZstdCompressor()
         with open(path, "rb") as input_f:
             with open(f"{path}.zst", "wb") as output_f:
                 cctx.copy_stream(input_f, output_f)
 
-    def download_db(self, db_type):
-        path = f"data/{db_type}.json"
-        formatted_base_url = BASE_URL.format(db_type, f"v{get_bugbug_version()}")
-        url = f"{formatted_base_url}/{db_type}.json.zst"
-        logger.info(f"Downloading {db_type} database from {url} to {path}.zst")
-        urlretrieve(url, f"{path}.zst")
-        assert os.path.exists(f"{path}.zst"), "Downloaded file exists"
-        logger.info(f"Decompressing {db_type} database")
-        self.decompress_file(path)
-
     def go(self, model_name):
         # Download datasets that were built by bugbug_data.
         os.makedirs("data", exist_ok=True)
@@ -54,10 +34,10 @@ def go(self, model_name):
             or isinstance(model_obj, model.BugCoupleModel)
             or (hasattr(model_obj, "bug_data") and model_obj.bug_data)
         ):
-            self.download_db("bugs")
+            db.download(bugzilla.BUGS_DB, force=True)
 
         if isinstance(model_obj, model.CommitModel):
-            self.download_db("commits")
+            db.download(repository.COMMITS_DB, force=True)
 
         logger.info(f"Training *{model_name}* model")
         metrics = model_obj.train()


Test patch:
Repository: mozilla/bugbug

Issue:
When calculating and showing feature importance, also show the values of the features
@chidauri you might be interested in this.


Discussion:

Resolving patch:
diff --git a/bugbug/model.py b/bugbug/model.py
--- a/bugbug/model.py
+++ b/bugbug/model.py
@@ -250,7 +250,12 @@ def print_feature_importances(
                 else:
                     shap_val.append("-" + str(importance))
 
-                top_feature_names.append(feature_names[int(index)])
+                feature_value = np.squeeze(
+                    important_features["values"].toarray()[:, int(index)]
+                )
+                top_feature_names.append(
+                    f"{feature_names[int(index)]} = {feature_value.round(decimals=5)}"
+                )
             shap_val = [[predicted_class] + shap_val]
 
         # extract importance values from the top features for all the classes
@@ -265,16 +270,16 @@ def print_feature_importances(
                 for class_name, imp_values in important_features["classes"].items()
             ]
 
-        # allow maximum of 6 columns in a row to fit the page better
+        # allow maximum of 5 columns in a row to fit the page better
         print("Top {} features:".format(len(top_feature_names)))
-        for i in range(0, len(top_feature_names), 6):
+        for i in range(0, len(top_feature_names), 5):
             table = []
             for item in shap_val:
-                table.append(item[i : i + 6])
+                table.append(item[i : i + 5])
             print(
                 tabulate(
                     table,
-                    headers=(["classes"] + top_feature_names)[i : i + 6],
+                    headers=(["classes"] + top_feature_names)[i : i + 5],
                     tablefmt="grid",
                 ),
                 end="\n\n",
@@ -476,6 +481,7 @@ def classify(
             important_features = self.get_important_features(
                 importance_cutoff, shap_values
             )
+            important_features["values"] = X
 
             # Workaround: handle multi class case for force_plot to work correctly
             if len(classes[0]) > 2:


Test patch:
Repository: mozilla/bugbug

Issue:
Add an option to the bug_classifier script to download the model when it doesn't exist



Discussion:
@marco-c `f"https://index.taskcluster.net/v1/task/project.relman.bugbug.train_{model_name}.latest/artifacts/public/{model_file_name}.zst"` is this the right url ?
Yes, for example https://index.taskcluster.net/v1/task/project.relman.bugbug.train_component.latest/artifacts/public/componentmodel.zst.
@marco-c for some models it is working fine(`stepstoreproduce`) but for some the zstdecompression isn't working :(

> upliftmodel does not exist. Downloading the model....
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/home/harshit/Documents/bugbug/scripts/bug_classifier.py", line 80, in main
    classify_bugs(args.model, args.classifier)
  File "/home/harshit/Documents/bugbug/scripts/bug_classifier.py", line 35, in classify_bugs
    dctx.copy_stream(input_f, output_f)
zstd.ZstdError: zstd decompressor error: Unknown frame descriptor

You need to check if the model is available at the URL, not all models are.

Resolving patch:
diff --git a/scripts/bug_classifier.py b/scripts/bug_classifier.py
--- a/scripts/bug_classifier.py
+++ b/scripts/bug_classifier.py
@@ -2,14 +2,20 @@
 
 import argparse
 import os
+from logging import INFO, basicConfig, getLogger
 
 import numpy as np
+import requests
 
 from bugbug import bugzilla
 from bugbug.models import get_model_class
+from bugbug.utils import download_check_etag, zstd_decompress
 
 MODELS_WITH_TYPE = ("component",)
 
+basicConfig(level=INFO)
+logger = getLogger(__name__)
+
 
 def classify_bugs(model_name, classifier):
     if classifier != "default":
@@ -22,9 +28,21 @@ def classify_bugs(model_name, classifier):
     else:
         model_file_name = f"{model_name}model"
 
-    assert os.path.exists(
-        model_file_name
-    ), f"{model_file_name} does not exist. Train the model with trainer.py first."
+    if not os.path.exists(model_file_name):
+        logger.info(f"{model_file_name} does not exist. Downloading the model....")
+        try:
+            download_check_etag(
+                f"https://index.taskcluster.net/v1/task/project.relman.bugbug.train_{model_name}.latest/artifacts/public/{model_file_name}.zst",
+                f"{model_file_name}.zst",
+            )
+        except requests.HTTPError:
+            logger.error(
+                f"A pre-trained model is not available, you will need to train it yourself using the trainer script"
+            )
+            raise SystemExit(1)
+
+        zstd_decompress(model_file_name)
+        assert os.path.exists(model_file_name), "Decompressed file doesn't exist"
 
     model_class = get_model_class(model_name)
     model = model_class.load(model_file_name)


Test patch:
Repository: mozilla/bugbug

Issue:
Rename spawn_data_pipeline.py to spawn_pipeline.py
Since it supports spawning both data and check (or any other pipeline defined in a yaml file).


Discussion:
can I claim this issue?
Sure, you can work on it! Any issue is unassigned until there is a PR to fix it.
is changing the name enough or do I have to make other relevant changes?

Just need to rename the file and update all places which are referencing it.
Hey, I completed this. Shall i create a branch and and then create a Pull Request?
Yes.
I'm getting this error:
"remote: Permission to mozilla/bugbug.git denied to hmrishavbandy." ,
when i am trying to push my changes. I am using the terminal and am on Ubuntu 19.04. 
You need to push them to your fork, not to the upstream repository.
Thanks a lot @marco-c  for all this help. You helped make this fun :D
@marco-c  Can I take up this issue,since there is no PR for it? 
marco-c  I am new to open source contribution . This issue is not closed yet . Can I take up this issue .

@marco-c I am new to open source contribution, and as this issue is not closed yet. Can I take this?

Hi All its weekend so Marco most probably is on leave, @hzk120 please go ahead and make the PR if you need any help comment here :)
@marco-c  If this issue is still open. Can I work on it ?
@Patil2099 read CONTRIBUTING.md. You can work on any open issue which has no PR linked to it.
This one doesn't have a open PR linked to it, so it's not assigned.

Resolving patch:
diff --git a/infra/spawn_data_pipeline.py b/infra/spawn_pipeline.py
similarity index 100%
rename from infra/spawn_data_pipeline.py
rename to infra/spawn_pipeline.py


Test patch:
Repository: mozilla/bugbug

Issue:
Add an option to the bug_classifier script to classify a specific bug



Discussion:

Resolving patch:
diff --git a/scripts/bug_classifier.py b/scripts/bug_classifier.py
--- a/scripts/bug_classifier.py
+++ b/scripts/bug_classifier.py
@@ -17,7 +17,7 @@
 logger = getLogger(__name__)
 
 
-def classify_bugs(model_name, classifier):
+def classify_bugs(model_name, classifier, bug_id):
     if classifier != "default":
         assert (
             model_name in MODELS_WITH_TYPE
@@ -47,7 +47,13 @@ def classify_bugs(model_name, classifier):
     model_class = get_model_class(model_name)
     model = model_class.load(model_file_name)
 
-    for bug in bugzilla.get_bugs():
+    if bug_id:
+        bugs = bugzilla.get(bug_id).values()
+        assert bugs, f"A bug with a bug id of {bug_id} was not found"
+    else:
+        bugs = bugzilla.get_bugs()
+
+    for bug in bugs:
         print(
             f'https://bugzilla.mozilla.org/show_bug.cgi?id={bug["id"]} - {bug["summary"]} '
         )
@@ -83,7 +89,8 @@ def main():
         choices=["default", "nn"],
         default="default",
     )
+    parser.add_argument("--bug-id", help="Classify the given bug id")
 
     args = parser.parse_args()
 
-    classify_bugs(args.model, args.classifier)
+    classify_bugs(args.model, args.classifier, args.bug_id)


Test patch:
Repository: mozilla/bugbug

Issue:
Error while using rollback for certain bugs
Looks like we missed an edge case. For a few bugs, `bugs["comments"] = []`, hence https://github.com/mozilla/bugbug/blob/master/bugbug/bug_snapshot.py#L837 gives an error. I'm wondering why we're handling the error later, in [line 861](https://github.com/mozilla/bugbug/blob/master/bugbug/bug_snapshot.py#L861), maybe this should be shifted up or we should add a new condition to handle these special bugs?





Discussion:
Yeah, I think you can just shift that part up.

Resolving patch:
diff --git a/bugbug/bug_snapshot.py b/bugbug/bug_snapshot.py
--- a/bugbug/bug_snapshot.py
+++ b/bugbug/bug_snapshot.py
@@ -833,6 +833,18 @@ def parse_flag_change(change):
 
                 bug[field] = old_value
 
+    if len(bug["comments"]) == 0:
+        assert_or_log("There must be at least one comment")
+        bug["comments"] = [
+            {
+                "count": 0,
+                "id": 0,
+                "text": "",
+                "author": bug["creator"],
+                "creation_time": bug["creation_time"],
+            }
+        ]
+
     # If the first comment is hidden.
     if bug["comments"][0]["count"] != 0:
         bug["comments"].insert(
@@ -858,17 +870,6 @@ def parse_flag_change(change):
         <= rollback_date
     ]
 
-    if len(bug["comments"]) == 0:
-        assert_or_log("There must be at least one comment")
-        bug["comments"] = [
-            {
-                "id": 0,
-                "text": "",
-                "author": bug["creator"],
-                "creation_time": bug["creation_time"],
-            }
-        ]
-
     return bug
 
 


Test patch:
Repository: mozilla/bugbug

Issue:
Support multiclass classifiers in bug_classifier script
The bug_classifier script at scripts/bug_classifier.py currently assumes the model is a binary model. We need to make it work for multiclass models too (e.g. defectenhancementtask).

In particular, https://github.com/mozilla/bugbug/blob/65bf1b4604ca55a67490d27adc99c6441bad38c8/scripts/bug_classifier.py#L75-L78 needs to be changed.

To test your changes, simply run `python3 -m scripts.bug_classifier defectenhancementtask`


Discussion:
Hi , I am new here. I would like to work on this. Can anyone guide me through?

I've expanded the description in the first comment.
Hi @marco-c , When I was installing the requirements in requirements.txt I get an error for rs_parsepatch. I am currently on Mac OS.X . Does parsepatch support OSX?

No: https://github.com/mozilla/pyo3-parsepatch/issues/4.
You don't need it though to fix this issue.
Can I work on this? I want to work on this issue, but as I am a beginner this is new to me. can you please guide me on how to proceed? Thanks:)
No need to ask, you can work on any issue that has no open PR linked to it!
The description of what needs to be done is in the first comment. The first step is to set-up the repository as described in README, then try to run `python3 -m scripts.bug_classifier defectenhancementtask`. It will output "Positive" or "Negative" according to the prediction made by the model, it should instead output the predicted class name.
Hello @marco-c , 
I followed the instruction to setup the project and after running `python3 -m scripts.bug_classifier defectenhancementtask` command, it not displaying any output. Any suggestion?

![IMG_20191109_173833](https://user-images.githubusercontent.com/18494493/68528381-ef2c2400-0317-11ea-94cc-783e84bc6efb.png)

It shouldn't happen, but please file a separate issue about this problem.

Resolving patch:
diff --git a/scripts/bug_classifier.py b/scripts/bug_classifier.py
--- a/scripts/bug_classifier.py
+++ b/scripts/bug_classifier.py
@@ -69,10 +69,13 @@ def classify_bugs(model_name, classifier, bug_id):
         else:
             probas = model.classify(bug, probabilities=True, importances=False)
 
-        if np.argmax(probas) == 1:
-            print(f"Positive! {probas}")
+        probability = probas[0]
+        pred_index = np.argmax(probability)
+        if len(probability) > 2:
+            pred_class = model.le.inverse_transform([pred_index])[0]
         else:
-            print(f"Negative! {probas}")
+            pred_class = "Positive" if pred_index == 1 else "Negative"
+        print(f"{pred_class} {probability}")
         input()
 
 


Test patch:
Repository: mozilla/bugbug

Issue:
Log number of spam/non-spam bugs in SpamBug get_labels



Discussion:
#1208 
@marco-c Please guide me if any changes needed.

Resolving patch:
diff --git a/bugbug/models/spambug.py b/bugbug/models/spambug.py
--- a/bugbug/models/spambug.py
+++ b/bugbug/models/spambug.py
@@ -85,6 +85,17 @@ def get_labels(self):
             ):
                 classes[bug_id] = 1
 
+        print(
+            "{} bugs are classified as non-spam".format(
+                sum(1 for label in classes.values() if label == 0)
+            )
+        )
+        print(
+            "{} bugs are classified as spam".format(
+                sum(1 for label in classes.values() if label == 1)
+            )
+        )
+
         return classes, [0, 1]
 
     def items_gen(self, classes):


Test patch:
Repository: mozilla/bugbug

Issue:
Set label as 0 in the QANeeded model when one of the qa flags are removed



Discussion:
Hey @marco-c, after exploring `bugbug/models/qaneeded.py` I came across this method `get_labels()` which I think needs the required modification. So, If I add a check for one of the QA flags in the `removed` field of a change and set the label accordingly then that should do it.

Can you please confirm that am I on the right track, and also If I'm missing something? That would really help me, thanks.

Resolving patch:
diff --git a/bugbug/models/qaneeded.py b/bugbug/models/qaneeded.py
--- a/bugbug/models/qaneeded.py
+++ b/bugbug/models/qaneeded.py
@@ -91,11 +91,18 @@ def get_labels(self):
             if not found_qa:
                 for entry in bug_data["history"]:
                     for change in entry["changes"]:
+                        if any(
+                            change["removed"].startswith(label)
+                            for label in ["qawanted", "qe-verify", "qaurgent"]
+                        ):
+                            classes[bug_id] = 0
+
                         if any(
                             change["added"].startswith(label)
                             for label in ["qawanted", "qe-verify", "qaurgent"]
                         ):
                             classes[bug_id] = 1
+
             if bug_id not in classes:
                 classes[bug_id] = 0
 


Test patch:
Repository: mozilla/bugbug

Issue:
Return 404 when a model which doesn't exist is requested
The HTTP service APIs (http_service/bugbug_http/app.py) should return 404 when the requested model is not supported (that is, not in the list at https://github.com/mozilla/bugbug/blob/e2a54a04ab69efd91c74a67805271009afb714a5/http_service/bugbug_http/models.py#L23).


Discussion:
Hey, I want to work on this issue. Can you please elaborate this issue please and where do I need to do changes?
I've updated the first comment to explain the issue in more detail.

Resolving patch:
diff --git a/http_service/bugbug_http/app.py b/http_service/bugbug_http/app.py
--- a/http_service/bugbug_http/app.py
+++ b/http_service/bugbug_http/app.py
@@ -262,6 +262,9 @@ def model_prediction(model_name, bug_id):
     else:
         LOGGER.info("Request with API TOKEN %r", auth)
 
+    if model_name not in MODELS_NAMES:
+        return jsonify({"error": f"Model {model_name} doesn't exist"}), 404
+
     # Get the latest change from Bugzilla for the bug
     bug = get_bugs_last_change_time([bug_id])
 
@@ -405,6 +408,9 @@ def batch_prediction(model_name):
     else:
         LOGGER.info("Request with API TOKEN %r", auth)
 
+    if model_name not in MODELS_NAMES:
+        return jsonify({"error": f"Model {model_name} doesn't exist"}), 404
+
     # TODO Check is JSON is valid and validate against a request schema
     batch_body = json.loads(request.data)
 


Test patch:
diff --git a/http_service/tests/test_app.py b/http_service/tests/test_app.py
--- a/http_service/tests/test_app.py
+++ b/http_service/tests/test_app.py
@@ -68,6 +68,21 @@ def test_non_int_batch(client):
     }
 
 
+def test_unknown_model(client):
+    """Start with  a blank database,"""
+    bugs = [1, 2, 3]
+    unknown_model = "unknown_model"
+
+    rv = client.post(
+        f"/{unknown_model}/predict/batch",
+        data=json.dumps({"bugs": bugs}),
+        headers={API_TOKEN: "test"},
+    )
+
+    assert rv.status_code == 404
+    assert rv.json == {"error": f"Model {unknown_model} doesn't exist"}
+
+
 def test_no_api_key(client):
     """Start with an empty database,"""
Repository: mozilla/bugbug

Issue:
'dict' object has no attribute 'data' while accessing error message from UnauthorizedError().dump({})
The following line throws a 'dict' object has no attribute 'data'
https://github.com/mozilla/bugbug/blob/1fb819b1317f15ac883c337d4313c955db9b4eeb/http_service/bugbug_http/app.py#L261

This is because the UnauthorizedError schema defines a field named  "message" and not "data"
https://github.com/mozilla/bugbug/blob/1fb819b1317f15ac883c337d4313c955db9b4eeb/http_service/bugbug_http/app.py#L84-L85

Found this while working on #1343 


Discussion:
Thanks for the report! @ahal is fixing this in #1319.
Awesome! This means that  I can work on #1343 only once this fix is issued right?
Please let me know in this thread once it's fixed. I'll  raise the PR for the test case accordingly.

Feel free to submit a PR to fix just the `.data` issue, and @ahal will rebase his PR on top of yours.

Resolving patch:
diff --git a/http_service/bugbug_http/app.py b/http_service/bugbug_http/app.py
--- a/http_service/bugbug_http/app.py
+++ b/http_service/bugbug_http/app.py
@@ -258,7 +258,7 @@ def model_prediction(model_name, bug_id):
     auth = headers.get(API_TOKEN)
 
     if not auth:
-        return jsonify(UnauthorizedError().dump({}).data), 401
+        return jsonify(UnauthorizedError().dump({})), 401
     else:
         LOGGER.info("Request with API TOKEN %r", auth)
 
@@ -401,7 +401,7 @@ def batch_prediction(model_name):
     auth = headers.get(API_TOKEN)
 
     if not auth:
-        return jsonify(UnauthorizedError().dump({}).data), 401
+        return jsonify(UnauthorizedError().dump({})), 401
     else:
         LOGGER.info("Request with API TOKEN %r", auth)
 


Test patch:
diff --git a/http_service/tests/test_app.py b/http_service/tests/test_app.py
--- a/http_service/tests/test_app.py
+++ b/http_service/tests/test_app.py
@@ -66,3 +66,16 @@ def test_non_int_batch(client):
             ]
         }
     }
+
+
+def test_no_api_key(client):
+    """Start with an empty database,"""
+
+    bugs = [1, 2, 3]
+
+    rv = client.post(
+        "/component/predict/batch", data=json.dumps({"bugs": bugs}), headers={},
+    )
+
+    assert rv.status_code == 401
+    assert rv.json == {"message": "Error, missing X-API-KEY"}