Repository: huggingface/huggingface_hub

Issue:
Add sentence similarity for spaCy
Computing similarity is simple with `spaCy` and there is an existing [template pipeline](https://github.com/huggingface/huggingface_hub/blob/main/api-inference-community/docker_images/common/app/pipelines/sentence_similarity.py) for sentence similarity at which can be modified for this. 

https://spacy.io/api/doc#similarity


Discussion:
Hi! @osanseviero I would like to contribute 
Hi @devpramod! Excellent! Contributions are always welcomed. I think this should be easier to introduce since there is already an existing template for this use case. Let me know if you have any questions

Resolving patch:
diff --git a/api-inference-community/docker_images/spacy/app/main.py b/api-inference-community/docker_images/spacy/app/main.py
--- a/api-inference-community/docker_images/spacy/app/main.py
+++ b/api-inference-community/docker_images/spacy/app/main.py
@@ -6,6 +6,7 @@
 from api_inference_community.routes import pipeline_route, status_ok
 from app.pipelines import (
     Pipeline,
+    SentenceSimilarityPipeline,
     TextClassificationPipeline,
     TokenClassificationPipeline,
 )
@@ -30,6 +31,7 @@
 # - translation
 # - summarization
 # - automatic-speech-recognition
+# - sentence-similarity
 # - ...
 # For instance
 # from app.pipelines import AutomaticSpeechRecognitionPipeline
@@ -39,6 +41,7 @@
 ALLOWED_TASKS: Dict[str, Type[Pipeline]] = {
     "token-classification": TokenClassificationPipeline,
     "text-classification": TextClassificationPipeline,
+    "sentence-similarity": SentenceSimilarityPipeline,
 }
 
 
diff --git a/api-inference-community/docker_images/spacy/app/pipelines/sentence_similarity.py b/api-inference-community/docker_images/spacy/app/pipelines/sentence_similarity.py
--- a/api-inference-community/docker_images/spacy/app/pipelines/sentence_similarity.py
+++ b/api-inference-community/docker_images/spacy/app/pipelines/sentence_similarity.py
@@ -1,3 +1,6 @@
+import os
+import subprocess
+import sys
 from typing import Dict, List, Union
 
 from app.pipelines import Pipeline
@@ -8,14 +11,23 @@ def __init__(
         self,
         model_id: str,
     ):
-        # IMPLEMENT_THIS
-        # Preload all the elements you are going to need at inference.
-        # For instance your model, processors, tokenizer that might be needed.
-        # This function is only called once, so do all the heavy processing I/O here
-        raise NotImplementedError(
-            "Please implement SentenceSimilarityPipeline __init__ function"
+        # At the time, only public models from spaCy are allowed in the inference API.
+        full_model_path = model_id.split("/")
+        if len(full_model_path) != 2:
+            raise ValueError(
+                f"Invalid model_id: {model_id}. It should have a namespace (:namespace:/:model_name:)"
+            )
+        namespace, model_name = full_model_path
+        package = f"https://huggingface.co/{namespace}/{model_name}/resolve/main/{model_name}-any-py3-none-any.whl"
+        cache_dir = os.environ["PIP_CACHE"]
+        subprocess.check_call(
+            [sys.executable, "-m", "pip", "install", "--cache-dir", cache_dir, package]
         )
 
+        import spacy
+
+        self.model = spacy.load(model_name)
+
     def __call__(self, inputs: Dict[str, Union[str, List[str]]]) -> List[float]:
         """
         Args:
@@ -27,7 +39,12 @@ def __call__(self, inputs: Dict[str, Union[str, List[str]]]) -> List[float]:
         Return:
             A :obj:`list` of floats: Some similarity measure between `source_sentence` and each sentence from `sentences`.
         """
-        # IMPLEMENT_THIS
-        raise NotImplementedError(
-            "Please implement SentenceSimilarityPipeline __call__ function"
-        )
+        source_sentence = inputs["source_sentence"]
+        source_doc = self.model(source_sentence)
+
+        similarities = []
+        for sentence in inputs["sentences"]:
+            search_doc = self.model(sentence)
+            similarities.append(source_doc.similarity(search_doc))
+
+        return similarities


Test patch:
diff --git a/api-inference-community/docker_images/spacy/tests/test_api.py b/api-inference-community/docker_images/spacy/tests/test_api.py
--- a/api-inference-community/docker_images/spacy/tests/test_api.py
+++ b/api-inference-community/docker_images/spacy/tests/test_api.py
@@ -14,6 +14,7 @@
     # "text-generation": "mysample-gpt2",
     "token-classification": "spacy/en_core_web_sm",
     "text-classification": "explosion/en_textcat_goemotions",
+    "sentence-similarity": "spacy/en_core_web_sm",
 }
 
 
diff --git a/api-inference-community/docker_images/spacy/tests/test_api_sentence_similarity.py b/api-inference-community/docker_images/spacy/tests/test_api_sentence_similarity.py
--- a/api-inference-community/docker_images/spacy/tests/test_api_sentence_similarity.py
+++ b/api-inference-community/docker_images/spacy/tests/test_api_sentence_similarity.py
@@ -22,6 +22,12 @@ def setUp(self):
 
         self.app = app
 
+    @classmethod
+    def setUpClass(cls):
+        from app.main import get_pipeline
+
+        get_pipeline.cache_clear()
+
     def tearDown(self):
         if self.old_model_id is not None:
             os.environ["MODEL_ID"] = self.old_model_id
@@ -44,7 +50,6 @@ def test_simple(self):
 
         with TestClient(self.app) as client:
             response = client.post("/", json={"inputs": inputs})
-
         self.assertEqual(
             response.status_code,
             200,
diff --git a/api-inference-community/tests/test_dockers.py b/api-inference-community/tests/test_dockers.py
--- a/api-inference-community/tests/test_dockers.py
+++ b/api-inference-community/tests/test_dockers.py
@@ -116,7 +116,12 @@ def test_spacy(self):
         self.framework_docker_test(
             "spacy",
             "text-classification",
-            "explosion/en_textcat_goemotions",
+            "cverluise/xx_cat_pateexx_md",
+        )
+        self.framework_docker_test(
+            "spacy",
+            "sentence-similarity",
+            "spacy/en_core_web_sm",
         )
         self.framework_invalid_test("spacy")