Repository: huggingface/huggingface_hub (python)

Issue:
`metadata_update()` duplicates metrics when updating a single field from `None`
### Describe the bug

When updating evaluation metadata with the `metadata_update()` function, it seems that updating a field value from `None` to some other value results in the metrics being duplicated. 

See [this Hub PR](https://huggingface.co/autoevaluate/binary-classification/discussions/75/files) for an example, where I've simply changed the value of `verify_token` from `None` to `1234`. One sees that e.g. `accuracy` is duplicated to include a new "evaluation" element where the `verify_token` field is not `None`.

Intuitively, I would expect that updating an attribute of `EvalResult` would simply update in-place.

### Reproduction

Here's the code snippet the produced the above PR (feel free to run it without fear :)):

```python
from huggingface_hub import model_info, metadata_update
from huggingface_hub.repocard_data import model_index_to_eval_results, eval_results_to_model_index

model_id = "autoevaluate/binary-classification"
metadata = model_info(model_id)
model_index = metadata.cardData["model-index"]
_, eval_results = model_index_to_eval_results(model_index)

# Change verify_token from `None` to dummy value
for eval_result in eval_results:
    if eval_result.verified is True:
        eval_result.verify_token = "1234"
        
new_metadata = {"model-index": eval_results_to_model_index(model_id, eval_results)}        
metadata_update(model_id, metadata=new_metadata, overwrite=True, create_pr=True, commit_message="Test update")
```

### Logs

_No response_

### System info

```shell
- huggingface_hub version: 0.11.0
- Platform: Linux-4.19.0-22-cloud-amd64-x86_64-with-glibc2.10
- Python version: 3.8.13
- Running in iPython ?: Yes
- iPython shell: ZMQInteractiveShell
- Running in notebook ?: Yes
- Running in Google Colab ?: No
- Token path ?: /home/lewis_huggingface_co/.huggingface/token
- Has saved token ?: True
- Who am I ?: lewtun
- Configured git credential helpers: store
- FastAI: N/A
- Tensorflow: N/A
- Torch: 1.11.0
- Jinja2: 3.1.2
- Graphviz: N/A
- Pydot: N/A
```


Pull request:
Fix metadata_update for verified evaluations
Fixes #1210 and #1208

## TODO

- [x] Add regression tests

Patch:
diff --git a/src/huggingface_hub/repocard.py b/src/huggingface_hub/repocard.py
index f88c932a..74c3deba 100644
--- a/src/huggingface_hub/repocard.py
+++ b/src/huggingface_hub/repocard.py
@@ -807,6 +807,8 @@ def metadata_update(
                                 )
                             result_found = True
                             existing_result.metric_value = new_result.metric_value
+                            if existing_result.verified is True:
+                                existing_result.verify_token = new_result.verify_token
                     if not result_found:
                         card.data.eval_results.append(new_result)
         else:
diff --git a/src/huggingface_hub/repocard_data.py b/src/huggingface_hub/repocard_data.py
index 843d6092..bc70766d 100644
--- a/src/huggingface_hub/repocard_data.py
+++ b/src/huggingface_hub/repocard_data.py
@@ -121,6 +121,17 @@ class EvalResult:
     # A JSON Web Token that is used to verify whether the metrics originate from Hugging Face's [evaluation service](https://huggingface.co/spaces/autoevaluate/model-evaluator) or not.
     verify_token: Optional[str] = None
 
+    @property
+    def unique_identifier(self) -> tuple:
+        """Returns a tuple that uniquely identifies this evaluation."""
+        return (
+            self.task_type,
+            self.dataset_type,
+            self.dataset_config,
+            self.dataset_split,
+            self.dataset_revision,
+        )
+
     def is_equal_except_value(self, other: "EvalResult") -> bool:
         """
         Return True if `self` and `other` describe exactly the same metric but with a
@@ -129,7 +140,9 @@ class EvalResult:
         for key, _ in self.__dict__.items():
             if key == "metric_value":
                 continue
-            if getattr(self, key) != getattr(other, key):
+            # For metrics computed by Hugging Face's evaluation service, `verify_token` is derived from `metric_value`,
+            # so we exclude it here in the comparison.
+            if key != "verify_token" and getattr(self, key) != getattr(other, key):
                 return False
         return True
 
@@ -514,24 +527,25 @@ def eval_results_to_model_index(
     # Here, we make a map of those pairs and the associated EvalResults.
     task_and_ds_types_map = defaultdict(list)
     for eval_result in eval_results:
-        task_and_ds_pair = (eval_result.task_type, eval_result.dataset_type)
-        task_and_ds_types_map[task_and_ds_pair].append(eval_result)
+        task_and_ds_types_map[eval_result.unique_identifier].append(eval_result)
 
     # Use the map from above to generate the model index data.
     model_index_data = []
-    for (task_type, dataset_type), results in task_and_ds_types_map.items():
+    for results in task_and_ds_types_map.values():
+        # All items from `results` share same metadata
+        sample_result = results[0]
         data = {
             "task": {
-                "type": task_type,
-                "name": results[0].task_name,
+                "type": sample_result.task_type,
+                "name": sample_result.task_name,
             },
             "dataset": {
-                "name": results[0].dataset_name,
-                "type": dataset_type,
-                "config": results[0].dataset_config,
-                "split": results[0].dataset_split,
-                "revision": results[0].dataset_revision,
-                "args": results[0].dataset_args,
+                "name": sample_result.dataset_name,
+                "type": sample_result.dataset_type,
+                "config": sample_result.dataset_config,
+                "split": sample_result.dataset_split,
+                "revision": sample_result.dataset_revision,
+                "args": sample_result.dataset_args,
             },
             "metrics": [
                 {


Test patch:
diff --git a/tests/fixtures/cards/sample_simple_model_index.md b/tests/fixtures/cards/sample_simple_model_index.md
index 15d26c70..c0e6ff9b 100644
--- a/tests/fixtures/cards/sample_simple_model_index.md
+++ b/tests/fixtures/cards/sample_simple_model_index.md
@@ -8,7 +8,7 @@ tags:
 datasets:
 - beans
 metrics:
-- acc
+- accuracy
 model-index:
 - name: my-cool-model
   results:
@@ -18,12 +18,25 @@ model-index:
       type: beans
       name: Beans
     metrics:
-    - type: acc
+    - type: accuracy
       value: 0.9
+  - task:
+      type: image-classification
+    dataset:
+      type: beans
+      name: Beans
+      config: default
+      split: test
+      revision: 5503434ddd753f426f4b38109466949a1217c2bb
+      args:
+        date: 20220120
+    metrics:
+    - type: f1
+      value: 0.66
 ---
 
 # my-cool-model
 
 ## Model description
 
-You can embed local or remote images using `![](...)`
+This is a test model card with multiple evaluations across different (dataset, metric) configurations.
diff --git a/tests/test_repocard.py b/tests/test_repocard.py
index f0352268..4f3a14c2 100644
--- a/tests/test_repocard.py
+++ b/tests/test_repocard.py
@@ -127,7 +127,7 @@ model-index:
       value: 0.2662102282047272
       name: Accuracy
       config: default
-      verified: false
+      verified: true
 ---
 """
 
@@ -248,7 +248,7 @@ class RepocardMetadataTest(unittest.TestCase):
             metrics_id="accuracy",
             metrics_value=0.2662102282047272,
             metrics_config="default",
-            metrics_verified=False,
+            metrics_verified=True,
             dataset_pretty_name="ReactionGIF",
             dataset_id="julien-c/reactiongif",
             dataset_config="default",
@@ -322,6 +322,20 @@ class RepocardMetadataUpdateTest(unittest.TestCase):
         updated_metadata = metadata_load(self.repo_path / self.REPO_NAME / "README.md")
         self.assertDictEqual(updated_metadata, new_metadata)
 
+    def test_update_verify_token(self):
+        """Tests whether updating the verification token updates in-place.
+
+        Regression test for https://github.com/huggingface/huggingface_hub/issues/1210
+        """
+        new_metadata = copy.deepcopy(self.existing_metadata)
+        new_metadata["model-index"][0]["results"][0]["metrics"][0][
+            "verifyToken"
+        ] = "1234"
+        metadata_update(self.repo_id, new_metadata, token=self._token, overwrite=True)
+        self.repo.git_pull()
+        updated_metadata = metadata_load(self.repo_path / self.REPO_NAME / "README.md")
+        self.assertDictEqual(updated_metadata, new_metadata)
+
     def test_metadata_update_upstream(self):
         new_metadata = copy.deepcopy(self.existing_metadata)
         new_metadata["model-index"][0]["results"][0]["metrics"][0]["value"] = 0.1
@@ -766,6 +780,16 @@ class ModelCardTest(TestCaseWithCapLog):
         )
         self.assertIsNone(card.data.eval_results)
 
+    def test_model_card_with_model_index(self):
+        """Test that loading a model card with multiple evaluations is consistent with `metadata_load`.
+
+        Regression test for https://github.com/huggingface/huggingface_hub/issues/1208
+        """
+        sample_path = SAMPLE_CARDS_DIR / "sample_simple_model_index.md"
+        card = ModelCard.load(sample_path)
+        metadata = metadata_load(sample_path)
+        self.assertDictEqual(card.data.to_dict(), metadata)
+
     def test_load_model_card_from_file(self):
         sample_path = SAMPLE_CARDS_DIR / "sample_simple.md"
         card = ModelCard.load(sample_path)
@@ -824,7 +848,7 @@ class ModelCardTest(TestCaseWithCapLog):
                         metric_value=0.2662102282047272,
                         metric_name="Accuracy",
                         metric_config="default",
-                        verified=False,
+                        verified=True,
                     ),
                 ],
                 model_name="RoBERTa fine-tuned on ReactionGIF",