Repository: deepset-ai/haystack

Issue:
Ability to run Ray Serve detached - to allow running multiple instances of the app (HA)
**Is your feature request related to a problem? Please describe.**
Currently every `RayPipeline` starts a non-detached Ray Serve instances by calling `serve.start()` (see https://github.com/deepset-ai/haystack/blob/master/haystack/pipelines/ray.py#L71), which means a second instance of the Haystack app cannot start as it will not be allowed starting a second Ray Serve instance. This means you can only run one instance of the Haystack app - no HA (High Availability).

I am recommending allowing the use of the `detached=True` option of Ray Serve - which would allow the use of more than one instances of the Haystack app all connecting to the same Ray Serve instance instead of erroring out by trying to start separate Ray Serve instances.

This will allow running the Haystack application in HA mode (eg more than one instances) while using the same Ray Serve for model serving, by allowing the same Ray Serve instance being used by multiple instances of the same Haystack app.

**Describe the solution you'd like**
I am recommending allowing the use of the `detached=True` option of Ray Serve by adding a new arg to the `RayPipeline.load_from_yaml` (and other) methods - which would allow the use of more than one instances of the Haystack app all connecting to the same Ray Serve instance instead of erroring out.

I am willing to provide a PR.

**Describe alternatives you've considered**
There is no good alternative for a HA Haystack application - except not running in HA, which isn't really an alternative per say.

**Additional context**
N/A



Discussion:

Resolving patch:
diff --git a/haystack/pipelines/ray.py b/haystack/pipelines/ray.py
--- a/haystack/pipelines/ray.py
+++ b/haystack/pipelines/ray.py
@@ -61,14 +61,20 @@ class RayPipeline(Pipeline):
     YAML definitions of Ray pipelines are validated at load. For more information, see [YAML File Definitions](https://haystack-website-git-fork-fstau-dev-287-search-deepset-overnice.vercel.app/components/pipelines#yaml-file-definitions).
     """
 
-    def __init__(self, address: str = None, ray_args: Optional[Dict[str, Any]] = None):
+    def __init__(
+        self,
+        address: str = None,
+        ray_args: Optional[Dict[str, Any]] = None,
+        serve_args: Optional[Dict[str, Any]] = None,
+    ):
         """
         :param address: The IP address for the Ray cluster. If set to `None`, a local Ray instance is started.
         :param kwargs: Optional parameters for initializing Ray.
+        :param serve_args: Optional parameters for initializing Ray Serve.
         """
         ray_args = ray_args or {}
         ray.init(address=address, **ray_args)
-        serve.start()
+        self._serve_controller_client = serve.start(**serve_args)
         super().__init__()
 
     @classmethod
@@ -80,6 +86,7 @@ def load_from_config(
         strict_version_check: bool = False,
         address: Optional[str] = None,
         ray_args: Optional[Dict[str, Any]] = None,
+        serve_args: Optional[Dict[str, Any]] = None,
     ):
         validate_config(pipeline_config, strict_version_check=strict_version_check)
 
@@ -87,7 +94,7 @@ def load_from_config(
         component_definitions = get_component_definitions(
             pipeline_config=pipeline_config, overwrite_with_env_variables=overwrite_with_env_variables
         )
-        pipeline = cls(address=address, ray_args=ray_args or {})
+        pipeline = cls(address=address, ray_args=ray_args or {}, serve_args=serve_args or {})
 
         for node_config in pipeline_definition["nodes"]:
             if pipeline.root_node is None:
@@ -121,6 +128,7 @@ def load_from_yaml(  # type: ignore
         address: Optional[str] = None,
         strict_version_check: bool = False,
         ray_args: Optional[Dict[str, Any]] = None,
+        serve_args: Optional[Dict[str, Any]] = None,
     ):
         """
         Load Pipeline from a YAML file defining the individual components and how they're tied together to form
@@ -170,6 +178,7 @@ def load_from_yaml(  # type: ignore
                                              variable 'MYDOCSTORE_PARAMS_INDEX=documents-2021' can be set. Note that an
                                              `_` sign must be used to specify nested hierarchical properties.
         :param address: The IP address for the Ray cluster. If set to None, a local Ray instance is started.
+        :param serve_args: Optional parameters for initializing Ray Serve.
         """
         pipeline_config = read_pipeline_config_from_yaml(path)
         return RayPipeline.load_from_config(
@@ -179,6 +188,7 @@ def load_from_yaml(  # type: ignore
             strict_version_check=strict_version_check,
             address=address,
             ray_args=ray_args,
+            serve_args=serve_args,
         )
 
     @classmethod


Test patch:
diff --git a/test/pipelines/test_ray.py b/test/pipelines/test_ray.py
--- a/test/pipelines/test_ray.py
+++ b/test/pipelines/test_ray.py
@@ -14,6 +14,7 @@ def shutdown_ray():
     try:
         import ray
 
+        ray.serve.shutdown()
         ray.shutdown()
     except:
         pass
@@ -21,14 +22,17 @@ def shutdown_ray():
 
 @pytest.mark.integration
 @pytest.mark.parametrize("document_store_with_docs", ["elasticsearch"], indirect=True)
-def test_load_pipeline(document_store_with_docs):
+@pytest.mark.parametrize("serve_detached", [True, False])
+def test_load_pipeline(document_store_with_docs, serve_detached):
     pipeline = RayPipeline.load_from_yaml(
         SAMPLES_PATH / "pipeline" / "ray.haystack-pipeline.yml",
         pipeline_name="ray_query_pipeline",
         ray_args={"num_cpus": 8},
+        serve_args={"detached": serve_detached},
     )
     prediction = pipeline.run(query="Who lives in Berlin?", params={"Retriever": {"top_k": 10}, "Reader": {"top_k": 3}})
 
+    assert pipeline._serve_controller_client._detached == serve_detached
     assert ray.serve.get_deployment(name="ESRetriever").num_replicas == 2
     assert ray.serve.get_deployment(name="Reader").num_replicas == 1
     assert prediction["query"] == "Who lives in Berlin?"