Repository: mozilla/bugbug

Issue:
Make NLP dependencies optional
Some dependencies are only needed in some images, we should make some dependencies optional so that they are only installed when necessary.

For example, Spacy is the biggest dependency, but it is currently not really used by any image.


Discussion:
Let's start with Spacy, Tensorflow and Gensim, which are currently not used in any image.

There is bugbug code that is trying to import them, so we should change it so that we don't fail if the modules are not available.

Not sure what's the best implementation for this.
We could set them up as extra dependencies on the bugbug package.

For dev environment: ```pip install bugbug[ml-lib-whatever]```
For docker build : ```cd /tmp/bugbug && pip install .[ml-lib-whatever]```
Sounds good.

Just to be sure, they are not used because the Docker images do not use part of bugbug who need these dependencies, right?
> Just to be sure, they are not used because the Docker images do not use part of bugbug who need these dependencies, right?

Yep. Those dependencies in particular are not used by any image at the moment (e.g. tensorflow would be used by the component_nn model, but we are not using it yet).

Instead, other dependencies are only used by some images. E.g. only the bug retriever images use 'requests', the training images don't use it.

Resolving patch:
diff --git a/bugbug/nlp.py b/bugbug/nlp.py
--- a/bugbug/nlp.py
+++ b/bugbug/nlp.py
@@ -3,17 +3,41 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import sys
 from collections import defaultdict
 from functools import lru_cache
 
 import numpy as np
-import spacy
-from gensim.models import KeyedVectors
 from sklearn.base import BaseEstimator, TransformerMixin
 from sklearn.feature_extraction.text import TfidfVectorizer
-from spacy.tokenizer import Tokenizer
 
-nlp = spacy.load("en_core_web_sm")
+HAS_OPTIONAL_DEPENDENCIES = False
+
+try:
+    import spacy
+
+    from spacy.tokenizer import Tokenizer
+    from gensim.models import KeyedVectors
+
+    HAS_OPTIONAL_DEPENDENCIES = True
+except ImportError:
+    pass
+
+try:
+    if HAS_OPTIONAL_DEPENDENCIES:
+        nlp = spacy.load("en_core_web_sm")
+except OSError:
+    msg = (
+        "Spacy model is missing, install it with: "
+        f"{sys.executable} -m spacy download en_core_web_sm"
+    )
+    print(msg, file=sys.stderr)
+
+OPT_MSG_MISSING = (
+    "Optional dependencies are missing, install them with: pip install bugbug[nlp]\n"
+    "You might need also to download the models with: "
+    f"{sys.executable} -m spacy download en_core_web_sm"
+)
 
 
 def spacy_token_lemmatizer(text):
@@ -25,6 +49,11 @@ def spacy_token_lemmatizer(text):
 
 class SpacyVectorizer(TfidfVectorizer):
     def __init__(self, *args, **kwargs):
+
+        # Detect when the Spacy optional dependency is missing
+        if not HAS_OPTIONAL_DEPENDENCIES:
+            raise NotImplementedError(OPT_MSG_MISSING)
+
         super().__init__(tokenizer=spacy_token_lemmatizer, *args, **kwargs)
 
 
@@ -37,6 +66,10 @@ def get_word_embeddings():
 
 class MeanEmbeddingTransformer(BaseEstimator, TransformerMixin):
     def __init__(self):
+        # Detect when the Gensim optional dependency are missing
+        if not HAS_OPTIONAL_DEPENDENCIES:
+            raise NotImplementedError(OPT_MSG_MISSING)
+
         self.model = get_word_embeddings()
         self.dim = len(self.model["if"])
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -37,6 +37,20 @@ def read_requirements(file_):
 with open(os.path.join(here, "VERSION")) as f:
     version = f.read().strip()
 
+# Read the extra requirements
+extras = ["nlp"]
+
+extras_require = {}
+
+for extra in extras:
+    extras_install, extra_links = read_requirements("extra-%s-requirements.txt" % extra)
+
+    # Merge the dependency links
+    dependency_links.extend(extra_links)
+
+    extras_require[extra] = extras_install
+
+
 setup(
     name="bugbug",
     version=version,
@@ -44,6 +58,7 @@ def read_requirements(file_):
     author="Marco Castelluccio",
     author_email="mcastelluccio@mozilla.com",
     install_requires=install_requires,
+    extras_require=extras_require,
     dependency_links=dependency_links,
     packages=find_packages(exclude=["contrib", "docs", "tests"]),
     include_package_data=True,
@@ -55,4 +70,9 @@ def read_requirements(file_):
             "bugbug-train = scripts.trainer:main",
         ]
     },
+    classifiers=[
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3 :: Only",
+        "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+    ],
 )


Test patch: