Repository: huggingface/huggingface_hub
Pull request #1874 (2023-11-29 13:21:16)

Support source in EvalResult for model cards
Related to https://github.com/huggingface/hub-docs/pull/1144 and https://github.com/huggingface/moon-landing/pull/8232 (private repo).

We are adding a new field `source` for each eval result in the model card metadata. This PR adds support to it in the `EvalResult` object (cc @clefourrier).

**Example:**

```py
from huggingface_hub import EvalResult, ModelCard, ModelCardData

result = EvalResult(
    task_type="image-classification",
    dataset_type="beans",
    dataset_name="Beans",
    metric_type="acc",
    metric_value=0.9,
    source_name="Open LLM Leaderboard",
    source_url="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard",
)
card = ModelCard.from_template(ModelCardData(model_name="cool_model", eval_results=[result]))
```

which translates to

```yaml
---
model-index:
- name: cool_model
  results:
  - task:
      type: image-classification
    dataset:
      name: Beans
      type: beans
    metrics:
    - type: acc
      value: 0.9
    source:
      name: Open LLM Leaderboard
      url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard
---
```

HuggingFaceDocBuilderDev: The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/huggingface_hub/pr_1874). All of your documentation changes will be reflected on that endpoint.

Patch:
diff --git a/src/huggingface_hub/hf_file_system.py b/src/huggingface_hub/hf_file_system.py
index 80777155..5f5817b4 100644
--- a/src/huggingface_hub/hf_file_system.py
+++ b/src/huggingface_hub/hf_file_system.py
@@ -376,10 +376,10 @@ class HfFileSystem(fsspec.AbstractFileSystem):
                 out.append(cache_path_info)
         return out
 
-    def glob(self, path, maxdepth=None, **kwargs):
+    def glob(self, path, **kwargs):
         # Set expand_info=False by default to get a x10 speed boost
         kwargs = {"expand_info": kwargs.get("detail", False), **kwargs}
-        return super().glob(path, maxdepth=maxdepth, **kwargs)
+        return super().glob(path, **kwargs)
 
     def find(
         self,
diff --git a/src/huggingface_hub/repocard_data.py b/src/huggingface_hub/repocard_data.py
index df6af763..cc6d5426 100644
--- a/src/huggingface_hub/repocard_data.py
+++ b/src/huggingface_hub/repocard_data.py
@@ -50,6 +50,10 @@ class EvalResult:
             Indicates whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not. Automatically computed by Hugging Face, do not set.
         verify_token (`str`, *optional*):
             A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not.
+        source_name (`str`, *optional*):
+            The name of the source of the evaluation result. Example: "Open LLM Leaderboard".
+        source_url (`str`, *optional*):
+            The URL of the source of the evaluation result. Example: "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard".
     """
 
     # Required
@@ -117,6 +121,14 @@ class EvalResult:
     # A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not.
     verify_token: Optional[str] = None
 
+    # The name of the source of the evaluation result.
+    # Example: Open LLM Leaderboard
+    source_name: Optional[str] = None
+
+    # The URL of the source of the evaluation result.
+    # Example: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard
+    source_url: Optional[str] = None
+
     @property
     def unique_identifier(self) -> tuple:
         """Returns a tuple that uniquely identifies this evaluation."""
@@ -142,6 +154,10 @@ class EvalResult:
                 return False
         return True
 
+    def __post_init__(self) -> None:
+        if self.source_name is not None and self.source_url is None:
+            raise ValueError("If `source_name` is provided, `source_url` must also be provided.")
+
 
 @dataclass
 class CardData:
@@ -556,6 +572,8 @@ def model_index_to_eval_results(model_index: List[Dict[str, Any]]) -> Tuple[str,
             dataset_split = result["dataset"].get("split")
             dataset_revision = result["dataset"].get("revision")
             dataset_args = result["dataset"].get("args")
+            source_name = result.get("source", {}).get("name")
+            source_url = result.get("source", {}).get("url")
 
             for metric in result["metrics"]:
                 metric_type = metric["type"]
@@ -582,6 +600,8 @@ def model_index_to_eval_results(model_index: List[Dict[str, Any]]) -> Tuple[str,
                     metric_config=metric_config,
                     verified=verified,
                     verify_token=verify_token,
+                    source_name=source_name,
+                    source_url=source_url,
                 )
                 eval_results.append(eval_result)
     return name, eval_results
@@ -636,7 +656,7 @@ def eval_results_to_model_index(model_name: str, eval_results: List[EvalResult])
 
     # Metrics are reported on a unique task-and-dataset basis.
     # Here, we make a map of those pairs and the associated EvalResults.
-    task_and_ds_types_map = defaultdict(list)
+    task_and_ds_types_map: Dict[Any, List[EvalResult]] = defaultdict(list)
     for eval_result in eval_results:
         task_and_ds_types_map[eval_result.unique_identifier].append(eval_result)
 
@@ -671,6 +691,13 @@ def eval_results_to_model_index(model_name: str, eval_results: List[EvalResult])
                 for result in results
             ],
         }
+        if sample_result.source_url is not None:
+            source = {
+                "url": sample_result.source_url,
+            }
+            if sample_result.source_name is not None:
+                source["name"] = sample_result.source_name
+            data["source"] = source
         model_index_data.append(data)
 
     # TODO - Check if there cases where this list is longer than one?


Test patch:
diff --git a/tests/test_repocard_data.py b/tests/test_repocard_data.py
index fb109fe7..2efd10d1 100644
--- a/tests/test_repocard_data.py
+++ b/tests/test_repocard_data.py
@@ -14,6 +14,7 @@ from huggingface_hub.repocard_data import (
 )
 
 
+OPEN_LLM_LEADERBOARD_URL = "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"
 DUMMY_METADATA_WITH_MODEL_INDEX = """
 language: en
 license: mit
@@ -36,6 +37,9 @@ model-index:
     metrics:
     - type: acc
       value: 0.9
+    source:
+      name: Open LLM Leaderboard
+      url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard
 """
 
 
@@ -77,6 +81,8 @@ class ModelCardDataTest(unittest.TestCase):
                 dataset_name="Beans",
                 metric_type="acc",
                 metric_value=0.9,
+                source_name="Open LLM Leaderboard",
+                source_url=OPEN_LLM_LEADERBOARD_URL,
             ),
         ]
 
@@ -124,6 +130,10 @@ class ModelCardDataTest(unittest.TestCase):
                                 "verifyToken": 1234,
                             }
                         ],
+                        "source": {
+                            "name": "Open LLM Leaderboard",
+                            "url": OPEN_LLM_LEADERBOARD_URL,
+                        },
                     },
                 ],
             }
@@ -132,13 +142,22 @@ class ModelCardDataTest(unittest.TestCase):
 
         self.assertEqual(len(eval_results), 3)
         self.assertEqual(model_name, "my-cool-model")
+
         self.assertEqual(eval_results[0].dataset_type, "cats_vs_dogs")
+        self.assertIsNone(eval_results[0].source_name)
+        self.assertIsNone(eval_results[0].source_url)
+
         self.assertEqual(eval_results[1].metric_type, "f1")
         self.assertEqual(eval_results[1].metric_value, 0.9)
+        self.assertIsNone(eval_results[1].source_name)
+        self.assertIsNone(eval_results[1].source_url)
+
         self.assertEqual(eval_results[2].task_type, "image-classification")
         self.assertEqual(eval_results[2].dataset_type, "beans")
         self.assertEqual(eval_results[2].verified, True)
         self.assertEqual(eval_results[2].verify_token, 1234)
+        self.assertEqual(eval_results[2].source_name, "Open LLM Leaderboard")
+        self.assertEqual(eval_results[2].source_url, OPEN_LLM_LEADERBOARD_URL)
 
     def test_card_data_requires_model_name_for_eval_results(self):
         with pytest.raises(ValueError, match="`eval_results` requires `model_name` to be set."):
@@ -192,6 +211,28 @@ class ModelCardDataTest(unittest.TestCase):
         data_dict = data.to_dict()
         self.assertEqual(data_dict["some_arbitrary_kwarg"], "some_value")
 
+    def test_eval_result_with_incomplete_source(self):
+        # Source url without name: ok
+        EvalResult(
+            task_type="image-classification",
+            dataset_type="beans",
+            dataset_name="Beans",
+            metric_type="acc",
+            metric_value=0.9,
+            source_url=OPEN_LLM_LEADERBOARD_URL,
+        )
+
+        # Source name without url: not ok
+        with self.assertRaises(ValueError):
+            EvalResult(
+                task_type="image-classification",
+                dataset_type="beans",
+                dataset_name="Beans",
+                metric_type="acc",
+                metric_value=0.9,
+                source_name="Open LLM Leaderboard",
+            )
+
 
 class DatasetCardDataTest(unittest.TestCase):
     def test_train_eval_index_keys_updated(self):