Repository: deepset-ai/haystack

Issue:
ElasticsearchDocumentStore.get_metadata_values_by_key isn't returning more than 10 values
**Describe the bug**
ElasticsearchDocumentStore.get_metadata_values_by_key isn't returning more than 10 docs. Only the most frequently occurring top 10 metadata values are shown

**Expected behavior**
Return all metadata values with counts 

**FAQ Check**
- [x] Have you had a look at [our new FAQ page](https://haystack.deepset.ai/overview/faq)?

**System:**
 - OS: Ubuntu
 - Haystack version (commit or version number):  1.4.0
 - DocumentStore: ElasticSearchDocumentStore




Discussion:
Hi @suneelmatham , thank you for bringing this issue to our attention. We'll look into it as soon as possible and get back to you.

Resolving patch:
diff --git a/haystack/document_stores/elasticsearch.py b/haystack/document_stores/elasticsearch.py
--- a/haystack/document_stores/elasticsearch.py
+++ b/haystack/document_stores/elasticsearch.py
@@ -502,7 +502,10 @@ def get_metadata_values_by_key(
         :param headers: Custom HTTP headers to pass to elasticsearch client (e.g. {'Authorization': 'Basic YWRtaW46cm9vdA=='})
                 Check out https://www.elastic.co/guide/en/elasticsearch/reference/current/http-clients.html for more information.
         """
-        body: dict = {"size": 0, "aggs": {"metadata_agg": {"terms": {"field": key}}}}
+        body: dict = {
+            "size": 0,
+            "aggs": {"metadata_agg": {"composite": {"sources": [{key: {"terms": {"field": key}}}]}}},
+        }
         if query:
             body["query"] = {
                 "bool": {
@@ -514,11 +517,23 @@ def get_metadata_values_by_key(
                 body["query"] = {"bool": {}}
             body["query"]["bool"].update({"filter": LogicalFilterClause.parse(filters).convert_to_elasticsearch()})
         result = self.client.search(body=body, index=index, headers=headers)
-        buckets = result["aggregations"]["metadata_agg"]["buckets"]
-        for bucket in buckets:
-            bucket["count"] = bucket.pop("doc_count")
-            bucket["value"] = bucket.pop("key")
-        return buckets
+
+        values = []
+        current_buckets = result["aggregations"]["metadata_agg"]["buckets"]
+        after_key = result["aggregations"]["metadata_agg"].get("after_key", False)
+        for bucket in current_buckets:
+            values.append({"value": bucket["key"][key], "count": bucket["doc_count"]})
+
+        # Only 10 results get returned at a time, so apply pagination
+        while after_key:
+            body["aggs"]["metadata_agg"]["composite"]["after"] = after_key
+            result = self.client.search(body=body, index=index, headers=headers)
+            current_buckets = result["aggregations"]["metadata_agg"]["buckets"]
+            after_key = result["aggregations"]["metadata_agg"].get("after_key", False)
+            for bucket in current_buckets:
+                values.append({"value": bucket["key"][key], "count": bucket["doc_count"]})
+
+        return values
 
     def write_documents(
         self,


Test patch:
diff --git a/test/document_stores/test_document_store.py b/test/document_stores/test_document_store.py
--- a/test/document_stores/test_document_store.py
+++ b/test/document_stores/test_document_store.py
@@ -1202,17 +1202,15 @@ def test_custom_embedding_field(document_store_type, tmp_path):
 
 @pytest.mark.parametrize("document_store", ["elasticsearch"], indirect=True)
 def test_get_meta_values_by_key(document_store: BaseDocumentStore):
-    documents = [
-        Document(content="Doc1", meta={"meta_key_1": "1", "meta_key_2": "11"}),
-        Document(content="Doc2", meta={"meta_key_1": "2", "meta_key_2": "22"}),
-        Document(content="Doc3", meta={"meta_key_1": "3", "meta_key_2": "33"}),
-    ]
+    documents = [Document(content=f"Doc{i}", meta={"meta_key_1": f"{i}", "meta_key_2": f"{i}{i}"}) for i in range(20)]
     document_store.write_documents(documents)
 
     # test without filters or query
     result = document_store.get_metadata_values_by_key(key="meta_key_1")
+    possible_values = [f"{i}" for i in range(20)]
+    assert len(result) == 20
     for bucket in result:
-        assert bucket["value"] in ["1", "2", "3"]
+        assert bucket["value"] in possible_values
         assert bucket["count"] == 1
 
     # test with filters but no query
diff --git a/test/pipelines/test_eval.py b/test/pipelines/test_eval.py
--- a/test/pipelines/test_eval.py
+++ b/test/pipelines/test_eval.py
@@ -654,7 +654,7 @@ def test_extractive_qa_eval_answer_scope(reader, retriever_with_docs):
     assert metrics["Retriever"]["ndcg"] == 0.5
     assert metrics["Reader"]["exact_match"] == 0.5
     assert metrics["Reader"]["f1"] == 0.5
-    assert metrics["Reader"]["sas"] == 0.5
+    assert metrics["Reader"]["sas"] == pytest.approx(0.5)
 
     metrics = eval_result.calculate_metrics(answer_scope="document_id_and_context")
 
@@ -666,7 +666,7 @@ def test_extractive_qa_eval_answer_scope(reader, retriever_with_docs):
     assert metrics["Retriever"]["ndcg"] == 0.5
     assert metrics["Reader"]["exact_match"] == 0.5
     assert metrics["Reader"]["f1"] == 0.5
-    assert metrics["Reader"]["sas"] == 0.5
+    assert metrics["Reader"]["sas"] == pytest.approx(0.5)
 
 
 @pytest.mark.parametrize("retriever_with_docs", ["tfidf"], indirect=True)