Repository: huggingface/huggingface_hub

Issue:
Expose a function to update the meta data in the readme
As discussed in https://github.com/huggingface/evaluate/issues/6 we would like a function to update the meta data in the readme of a repo:

```python
huggingface_hub.metadata_update(
  repo_id="lvwerra/my_model",
  update={"metrics": new_value}
)
```

This would enable to programmatically update meta data such as metrics from the `evaluate`, `datasets`, or `transformers` libraries.

There are two use-cases:
1. The key does not exist yet so we add an entry.
2. The key does already exist but needs to be updated - in that case we might want a `overwrite=True` option as safety. 

What do you think @julien-c, @osanseviero? I can have a look at this if you want.


Discussion:
sounds good to me! 

this can use the existing metadata-related functions under the hood, and maybe rely on upload_file so it doesn't need a local install of `git`
Yes, I think using single file download + `metadata_load`/`metadata_save` (https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/repocard.py) + `upload_file` would be very good idea

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -15,7 +15,7 @@ def get_version() -> str:
     "filelock",
     "requests",
     "tqdm",
-    "pyyaml",
+    "pyyaml>=5.1",
     "typing-extensions>=3.7.4.3",  # to be able to import TypeAlias
     "importlib_metadata;python_version<'3.8'",
     "packaging>=20.9",
diff --git a/src/huggingface_hub/__init__.py b/src/huggingface_hub/__init__.py
--- a/src/huggingface_hub/__init__.py
+++ b/src/huggingface_hub/__init__.py
@@ -73,6 +73,12 @@
     push_to_hub_keras,
     save_pretrained_keras,
 )
+from .repocard import (
+    metadata_eval_result,
+    metadata_load,
+    metadata_save,
+    metadata_update,
+)
 from .repository import Repository
 from .snapshot_download import snapshot_download
 from .utils import logging
diff --git a/src/huggingface_hub/repocard.py b/src/huggingface_hub/repocard.py
--- a/src/huggingface_hub/repocard.py
+++ b/src/huggingface_hub/repocard.py
@@ -5,6 +5,8 @@
 from typing import Any, Dict, Optional, Union
 
 import yaml
+from huggingface_hub.file_download import hf_hub_download
+from huggingface_hub.hf_api import HfApi
 from huggingface_hub.repocard_types import (
     ModelIndex,
     SingleMetric,
@@ -13,10 +15,15 @@
     SingleResultTask,
 )
 
+from .constants import REPOCARD_NAME
+
 
 # exact same regex as in the Hub server. Please keep in sync.
 REGEX_YAML_BLOCK = re.compile(r"---[\n\r]+([\S\s]*?)[\n\r]+---[\n\r]")
 
+UNIQUE_RESULT_FEATURES = ["dataset", "task"]
+UNIQUE_METRIC_FEATURES = ["name", "type"]
+
 
 def metadata_load(local_path: Union[str, Path]) -> Optional[Dict]:
     content = Path(local_path).read_text()
@@ -99,3 +106,178 @@ def metadata_eval_result(
         model_index, dict_factory=lambda x: {k: v for (k, v) in x if v is not None}
     )
     return {"model-index": [data]}
+
+
+def metadata_update(
+    repo_id: str,
+    metadata: Dict,
+    *,
+    repo_type: str = None,
+    overwrite: bool = False,
+    token: str = None,
+) -> str:
+    """
+    Updates the metadata in the README.md of a repository on the Hugging Face Hub.
+
+    Example:
+    >>> from huggingface_hub import metadata_update
+    >>> metadata = {'model-index': [{'name': 'RoBERTa fine-tuned on ReactionGIF',
+    ...             'results': [{'dataset': {'name': 'ReactionGIF',
+    ...                                      'type': 'julien-c/reactiongif'},
+    ...                           'metrics': [{'name': 'Recall',
+    ...                                        'type': 'recall',
+    ...                                        'value': 0.7762102282047272}],
+    ...                          'task': {'name': 'Text Classification',
+    ...                                   'type': 'text-classification'}}]}]}
+    >>> update_metdata("julien-c/reactiongif-roberta", metadata)
+
+    Args:
+        repo_id (`str`):
+            The name of the repository.
+        metadata (`dict`):
+            A dictionary containing the metadata to be updated.
+        repo_type (`str`, *optional*):
+            Set to `"dataset"` or `"space"` if updating to a dataset or space,
+            `None` or `"model"` if updating to a model. Default is `None`.
+        overwrite (`bool`, *optional*, defaults to `False`):
+            If set to `True` an existing field can be overwritten, otherwise
+            attempting to overwrite an existing field will cause an error.
+        token (`str`, *optional*):
+            The Hugging Face authentication token.
+
+    Returns:
+        `str`: URL of the commit which updated the card metadata.
+    """
+
+    filepath = hf_hub_download(
+        repo_id,
+        filename=REPOCARD_NAME,
+        repo_type=repo_type,
+        use_auth_token=token,
+        force_download=True,
+    )
+    existing_metadata = metadata_load(filepath)
+
+    for key in metadata:
+        # update model index containing the evaluation results
+        if key == "model-index":
+            if "model-index" not in existing_metadata:
+                existing_metadata["model-index"] = metadata["model-index"]
+            else:
+                # the model-index contains a list of results as used by PwC but only has one element thus we take the first one
+                existing_metadata["model-index"][0][
+                    "results"
+                ] = _update_metadata_model_index(
+                    existing_metadata["model-index"][0]["results"],
+                    metadata["model-index"][0]["results"],
+                    overwrite=overwrite,
+                )
+        # update all fields except model index
+        else:
+            if key in existing_metadata and not overwrite:
+                if existing_metadata[key] != metadata[key]:
+                    raise ValueError(
+                        f"""You passed a new value for the existing meta data field '{key}'. Set `overwrite=True` to overwrite existing metadata."""
+                    )
+            else:
+                existing_metadata[key] = metadata[key]
+
+    # save and push to hub
+    metadata_save(filepath, existing_metadata)
+
+    return HfApi().upload_file(
+        path_or_fileobj=filepath,
+        path_in_repo=REPOCARD_NAME,
+        repo_id=repo_id,
+        repo_type=repo_type,
+        identical_ok=False,
+        token=token,
+    )
+
+
+def _update_metadata_model_index(existing_results, new_results, overwrite=False):
+    """
+    Updates the model-index fields in the metadata. If results with same unique
+    features exist they are updated, else a new result is appended. Updating existing
+    values is only possible if `overwrite=True`.
+
+    Args:
+        new_metrics (`List[dict]`):
+            List of new metadata results.
+        existing_metrics (`List[dict]`):
+            List of existing metadata results.
+        overwrite (`bool`, *optional*, defaults to `False`):
+            If set to `True`, an existing metric values can be overwritten, otherwise
+            attempting to overwrite an existing field will cause an error.
+
+    Returns:
+        `list`: List of updated metadata results
+    """
+    for new_result in new_results:
+        result_found = False
+        for existing_result_index, existing_result in enumerate(existing_results):
+            if all(
+                new_result[feat] == existing_result[feat]
+                for feat in UNIQUE_RESULT_FEATURES
+            ):
+                result_found = True
+                existing_results[existing_result_index][
+                    "metrics"
+                ] = _update_metadata_results_metric(
+                    new_result["metrics"],
+                    existing_result["metrics"],
+                    overwrite=overwrite,
+                )
+        if not result_found:
+            existing_results.append(new_result)
+    return existing_results
+
+
+def _update_metadata_results_metric(new_metrics, existing_metrics, overwrite=False):
+    """
+    Updates the metrics list of a result in the metadata. If metrics with same unique
+    features exist their values are updated, else a new metric is appended. Updating
+    existing values is only possible if `overwrite=True`.
+
+    Args:
+        new_metrics (`list`):
+            List of new metrics.
+        existing_metrics (`list`):
+            List of existing metrics.
+        overwrite (`bool`, *optional*, defaults to `False`):
+            If set to `True`, an existing metric values can be overwritten, otherwise
+            attempting to overwrite an existing field will cause an error.
+
+    Returns:
+        `list`: List of updated metrics
+    """
+    for new_metric in new_metrics:
+        metric_exists = False
+        for existing_metric_index, existing_metric in enumerate(existing_metrics):
+            if all(
+                new_metric[feat] == existing_metric[feat]
+                for feat in UNIQUE_METRIC_FEATURES
+            ):
+                if overwrite:
+                    existing_metrics[existing_metric_index]["value"] = new_metric[
+                        "value"
+                    ]
+                else:
+                    # if metric exists and value is not the same throw an error without overwrite flag
+                    if (
+                        existing_metrics[existing_metric_index]["value"]
+                        != new_metric["value"]
+                    ):
+                        existing_str = ", ".join(
+                            f"{feat}: {new_metric[feat]}"
+                            for feat in UNIQUE_METRIC_FEATURES
+                        )
+                        raise ValueError(
+                            "You passed a new value for the existing metric"
+                            f" '{existing_str}'. Set `overwrite=True` to overwrite"
+                            " existing metrics."
+                        )
+                metric_exists = True
+        if not metric_exists:
+            existing_metrics.append(new_metric)
+    return existing_metrics


Test patch:
diff --git a/tests/test_repocard.py b/tests/test_repocard.py
--- a/tests/test_repocard.py
+++ b/tests/test_repocard.py
@@ -11,16 +11,31 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import copy
 import os
 import shutil
+import tempfile
+import time
 import unittest
+import uuid
 from pathlib import Path
 
+import pytest
+
+import yaml
+from huggingface_hub import (
+    metadata_eval_result,
+    metadata_load,
+    metadata_save,
+    metadata_update,
+)
 from huggingface_hub.constants import REPOCARD_NAME
-from huggingface_hub.repocard import metadata_eval_result, metadata_load, metadata_save
+from huggingface_hub.hf_api import HfApi
+from huggingface_hub.repository import Repository
 from huggingface_hub.utils import logging
 
-from .testing_utils import set_write_permission_and_retry
+from .testing_constants import ENDPOINT_STAGING, TOKEN, USER
+from .testing_utils import retry_endpoint, set_write_permission_and_retry
 
 
 ROUND_TRIP_MODELCARD_CASE = """
@@ -98,6 +113,10 @@
 )
 
 
+def repo_name(id=uuid.uuid4().hex[:6]):
+    return "dummy-hf-hub-{0}-{1}".format(id, int(time.time() * 10e3))
+
+
 class RepocardTest(unittest.TestCase):
     def setUp(self):
         os.makedirs(REPOCARD_DIR, exist_ok=True)
@@ -159,3 +178,154 @@ def test_metadata_eval_result(self):
         metadata_save(filepath, data)
         content = filepath.read_text().splitlines()
         self.assertEqual(content, DUMMY_MODELCARD_EVAL_RESULT.splitlines())
+
+
+class RepocardUpdateTest(unittest.TestCase):
+    _api = HfApi(endpoint=ENDPOINT_STAGING)
+
+    @classmethod
+    def setUpClass(cls):
+        """
+        Share this valid token in all tests below.
+        """
+        cls._token = TOKEN
+        cls._api.set_access_token(TOKEN)
+
+    @retry_endpoint
+    def setUp(self) -> None:
+        self.repo_path = Path(tempfile.mkdtemp())
+        self.REPO_NAME = repo_name()
+        self.repo = Repository(
+            self.repo_path / self.REPO_NAME,
+            clone_from=f"{USER}/{self.REPO_NAME}",
+            use_auth_token=self._token,
+            git_user="ci",
+            git_email="ci@dummy.com",
+        )
+
+        with self.repo.commit("Add README to main branch"):
+            with open("README.md", "w+") as f:
+                f.write(DUMMY_MODELCARD_EVAL_RESULT)
+
+        self.existing_metadata = yaml.safe_load(
+            DUMMY_MODELCARD_EVAL_RESULT.strip().strip("---")
+        )
+
+    def tearDown(self) -> None:
+        self._api.delete_repo(repo_id=f"{self.REPO_NAME}", token=self._token)
+        shutil.rmtree(self.repo_path)
+
+    def test_update_dataset_name(self):
+        new_datasets_data = {"datasets": "['test/test_dataset']"}
+        metadata_update(
+            f"{USER}/{self.REPO_NAME}", new_datasets_data, token=self._token
+        )
+
+        self.repo.git_pull()
+        updated_metadata = metadata_load(self.repo_path / self.REPO_NAME / "README.md")
+        expected_metadata = copy.deepcopy(self.existing_metadata)
+        expected_metadata.update(new_datasets_data)
+        self.assertDictEqual(updated_metadata, expected_metadata)
+
+    def test_update_existing_result_with_overwrite(self):
+        new_metadata = copy.deepcopy(self.existing_metadata)
+        new_metadata["model-index"][0]["results"][0]["metrics"][0][
+            "value"
+        ] = 0.2862102282047272
+        metadata_update(
+            f"{USER}/{self.REPO_NAME}", new_metadata, token=self._token, overwrite=True
+        )
+
+        self.repo.git_pull()
+        updated_metadata = metadata_load(self.repo_path / self.REPO_NAME / "README.md")
+        self.assertDictEqual(updated_metadata, new_metadata)
+
+    def test_update_existing_result_without_overwrite(self):
+        new_metadata = copy.deepcopy(self.existing_metadata)
+        new_metadata["model-index"][0]["results"][0]["metrics"][0][
+            "value"
+        ] = 0.2862102282047272
+
+        with pytest.raises(
+            ValueError,
+            match=(
+                "You passed a new value for the existing metric 'name: Accuracy, type:"
+                " accuracy'. Set `overwrite=True` to overwrite existing metrics."
+            ),
+        ):
+            metadata_update(
+                f"{USER}/{self.REPO_NAME}",
+                new_metadata,
+                token=self._token,
+                overwrite=False,
+            )
+
+    def test_update_existing_field_without_overwrite(self):
+        new_datasets_data = {"datasets": "['test/test_dataset']"}
+        metadata_update(
+            f"{USER}/{self.REPO_NAME}", new_datasets_data, token=self._token
+        )
+
+        with pytest.raises(
+            ValueError,
+            match=(
+                "You passed a new value for the existing meta data field 'datasets'."
+                " Set `overwrite=True` to overwrite existing metadata."
+            ),
+        ):
+            new_datasets_data = {"datasets": "['test/test_dataset_2']"}
+            metadata_update(
+                f"{USER}/{self.REPO_NAME}",
+                new_datasets_data,
+                token=self._token,
+                overwrite=False,
+            )
+
+    def test_update_new_result_existing_dataset(self):
+        new_result = metadata_eval_result(
+            model_pretty_name="RoBERTa fine-tuned on ReactionGIF",
+            task_pretty_name="Text Classification",
+            task_id="text-classification",
+            metrics_pretty_name="Recall",
+            metrics_id="recall",
+            metrics_value=0.7762102282047272,
+            dataset_pretty_name="ReactionGIF",
+            dataset_id="julien-c/reactiongif",
+        )
+
+        metadata_update(
+            f"{USER}/{self.REPO_NAME}", new_result, token=self._token, overwrite=False
+        )
+
+        expected_metadata = copy.deepcopy(self.existing_metadata)
+        expected_metadata["model-index"][0]["results"][0]["metrics"].append(
+            new_result["model-index"][0]["results"][0]["metrics"][0]
+        )
+
+        self.repo.git_pull()
+        updated_metadata = metadata_load(self.repo_path / self.REPO_NAME / "README.md")
+        self.assertDictEqual(updated_metadata, expected_metadata)
+
+    def test_update_new_result_new_dataset(self):
+        new_result = metadata_eval_result(
+            model_pretty_name="RoBERTa fine-tuned on ReactionGIF",
+            task_pretty_name="Text Classification",
+            task_id="text-classification",
+            metrics_pretty_name="Accuracy",
+            metrics_id="accuracy",
+            metrics_value=0.2662102282047272,
+            dataset_pretty_name="ReactionJPEG",
+            dataset_id="julien-c/reactionjpeg",
+        )
+
+        metadata_update(
+            f"{USER}/{self.REPO_NAME}", new_result, token=self._token, overwrite=False
+        )
+
+        expected_metadata = copy.deepcopy(self.existing_metadata)
+        expected_metadata["model-index"][0]["results"].append(
+            new_result["model-index"][0]["results"][0]
+        )
+        self.repo.git_pull()
+        updated_metadata = metadata_load(self.repo_path / self.REPO_NAME / "README.md")
+        self.assertDictEqual(updated_metadata, expected_metadata)