Repository: mozilla/bugbug

Issue:
Store imbalanced metrics too



Discussion:

Resolving patch:
diff --git a/bugbug/model.py b/bugbug/model.py
--- a/bugbug/model.py
+++ b/bugbug/model.py
@@ -7,11 +7,17 @@
 
 import numpy as np
 import shap
-from imblearn.metrics import classification_report_imbalanced
+from imblearn.metrics import (
+    classification_report_imbalanced,
+    geometric_mean_score,
+    make_index_balanced_accuracy,
+    specificity_score,
+)
 from imblearn.pipeline import make_pipeline
 from sklearn import metrics
 from sklearn.externals import joblib
 from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.classification import precision_recall_fscore_support
 from sklearn.model_selection import cross_validate, train_test_split
 
 from bugbug import bugzilla, repository
@@ -19,6 +25,69 @@
 from bugbug.utils import split_tuple_iterator
 
 
+def classification_report_imbalanced_values(
+    y_true, y_pred, labels, target_names=None, sample_weight=None, digits=2, alpha=0.1
+):
+    """Copy of imblearn.metrics.classification_report_imbalanced to have
+    access to the raw values. The code is mostly the same except the
+    formatting code and generation of the report which haven removed. Copied
+    from version 0.4.3. The original code is living here:
+    https://github.com/scikit-learn-contrib/imbalanced-learn/blob/master/imblearn/metrics/_classification.py#L750
+
+
+    """
+    labels = np.asarray(labels)
+
+    if target_names is None:
+        target_names = ["%s" % l for l in labels]
+
+    # Compute the different metrics
+    # Precision/recall/f1
+    precision, recall, f1, support = precision_recall_fscore_support(
+        y_true, y_pred, labels=labels, average=None, sample_weight=sample_weight
+    )
+    # Specificity
+    specificity = specificity_score(
+        y_true, y_pred, labels=labels, average=None, sample_weight=sample_weight
+    )
+    # Geometric mean
+    geo_mean = geometric_mean_score(
+        y_true, y_pred, labels=labels, average=None, sample_weight=sample_weight
+    )
+    # Index balanced accuracy
+    iba_gmean = make_index_balanced_accuracy(alpha=alpha, squared=True)(
+        geometric_mean_score
+    )
+    iba = iba_gmean(
+        y_true, y_pred, labels=labels, average=None, sample_weight=sample_weight
+    )
+
+    result = {"targets": {}}
+
+    for i, label in enumerate(labels):
+        result["targets"][target_names[i]] = {
+            "precision": precision[i],
+            "recall": recall[i],
+            "specificity": specificity[i],
+            "f1": f1[i],
+            "geo_mean": geo_mean[i],
+            "iba": iba[i],
+            "support": support[i],
+        }
+
+    result["average"] = {
+        "precision": np.average(precision, weights=support),
+        "recall": np.average(recall, weights=support),
+        "specificity": np.average(specificity, weights=support),
+        "f1": np.average(f1, weights=support),
+        "geo_mean": np.average(geo_mean, weights=support),
+        "iba": np.average(iba, weights=support),
+        "support": np.sum(support),
+    }
+
+    return result
+
+
 class Model:
     def __init__(self, lemmatization=False):
         if lemmatization:
@@ -137,6 +206,7 @@ def train(self, importance_cutoff=0.15):
                     f'{i + 1}. \'{feature_names[int(index)]}\' ({"+" if (is_positive) else "-"}{importance})'
                 )
 
+        print("Test Set scores:")
         # Evaluate results on the test set.
         y_pred = self.clf.predict(X_test)
 
@@ -146,6 +216,11 @@ def train(self, importance_cutoff=0.15):
         tracking_metrics["confusion_matrix"] = confusion_matrix.tolist()
 
         print(classification_report_imbalanced(y_test, y_pred, labels=class_names))
+        report = classification_report_imbalanced_values(
+            y_test, y_pred, labels=class_names
+        )
+
+        tracking_metrics["report"] = report
 
         # Evaluate results on the test set for some confidence thresholds.
         for confidence_threshold in [0.6, 0.7, 0.8, 0.9]:
diff --git a/bugbug/utils.py b/bugbug/utils.py
--- a/bugbug/utils.py
+++ b/bugbug/utils.py
@@ -4,6 +4,7 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import collections
+import json
 import os
 import time
 
@@ -155,3 +156,16 @@ def retry(operation, retries=5, wait_between_retries=30):
                 raise
 
             time.sleep(wait_between_retries)
+
+
+class CustomJsonEncoder(json.JSONEncoder):
+    """ A custom Json Encoder to support Numpy types
+    """
+
+    def default(self, obj):
+        try:
+            return np.asscalar(obj)
+        except (ValueError, IndexError, AttributeError, TypeError):
+            pass
+
+        return super().default(self, obj)
diff --git a/scripts/trainer.py b/scripts/trainer.py
--- a/scripts/trainer.py
+++ b/scripts/trainer.py
@@ -10,6 +10,7 @@
 
 from bugbug import get_bugbug_version, model
 from bugbug.models import get_model_class
+from bugbug.utils import CustomJsonEncoder
 
 basicConfig(level=INFO)
 logger = getLogger(__name__)
@@ -63,7 +64,7 @@ def go(self, model_name):
         # Save the metrics as a file that can be uploaded as an artifact.
         metric_file_path = "metrics.json"
         with open(metric_file_path, "w") as metric_file:
-            json.dump(metrics, metric_file)
+            json.dump(metrics, metric_file, cls=CustomJsonEncoder)
 
         logger.info(f"Training done")
 


Test patch: