Repository: huggingface/huggingface_hub
Pull request #917 (2022-06-20 15:40:24)

[Keras Mixin] Rendering the Hyperparameter table vertically
This PR houses code for rendering the hyperparameter of the Keras models vertically.

The model card with the changes would look something like this.
![image](https://user-images.githubusercontent.com/36856589/174637246-c136e7ef-9da3-4f8e-ba50-9f61d4ae78fd.png)

The model card: https://huggingface.co/ariG23498/vertical-table-keras


HuggingFaceDocBuilderDev: The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/huggingface_hub/pr_917). All of your documentation changes will be reflected on that endpoint.

Patch:
diff --git a/src/huggingface_hub/keras_mixin.py b/src/huggingface_hub/keras_mixin.py
index 9d54135e..3ec95eba 100644
--- a/src/huggingface_hub/keras_mixin.py
+++ b/src/huggingface_hub/keras_mixin.py
@@ -33,18 +33,9 @@ def _create_hyperparameter_table(model):
         optimizer_params[
             "training_precision"
         ] = tf.keras.mixed_precision.global_policy().name
-        table = "|"
-        for key in optimizer_params.keys():
-            table += f" {key} |"
-
-        table += "\n|"
-        for key in optimizer_params.keys():
-            table += "-" * len(key)
-            table += "|"
-
-        table += "\n|"
-        for key in optimizer_params.keys():
-            table += f"{optimizer_params[key]}|"
+        table = "| Hyperparameters | Value |\n| :-- | :-- |\n"
+        for key, value in optimizer_params.items():
+            table += f"| {key} | {value} |\n"
     else:
         table = None
     return table
diff --git a/src/huggingface_hub/repocard.py b/src/huggingface_hub/repocard.py
index 6bf53857..9c723252 100644
--- a/src/huggingface_hub/repocard.py
+++ b/src/huggingface_hub/repocard.py
@@ -86,7 +86,94 @@ def metadata_eval_result(
     metrics_value: Any,
     dataset_pretty_name: str,
     dataset_id: str,
+    metrics_config: Optional[str] = None,
+    metrics_verified: Optional[bool] = False,
+    dataset_config: Optional[str] = None,
+    dataset_split: Optional[str] = None,
+    dataset_revision: Optional[str] = None,
 ) -> Dict:
+    """
+    Creates a metadata dict with the result from a model evaluated on a dataset.
+
+    Args:
+        model_pretty_name (`str`):
+            The name of the model in natural language.
+        task_pretty_name (`str`):
+            The name of a task in natural language.
+        task_id (`str`):
+            Example: automatic-speech-recognition. A task id.
+        metrics_pretty_name (`str`):
+            A name for the metric in natural language. Example: Test WER.
+        metrics_id (`str`):
+            Example: wer. A metric id from https://hf.co/metrics.
+        metrics_value (`Any`):
+            The value from the metric. Example: 20.0 or "20.0 ± 1.2".
+        dataset_pretty_name (`str`):
+            The name of the dataset in natural language.
+        dataset_id (`str`):
+            Example: common_voice. A dataset id from https://hf.co/datasets.
+        metrics_config (`str`, *optional*):
+            The name of the metric configuration used in `load_metric()`.
+            Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`.
+        metrics_verified (`bool`, *optional*, defaults to `False`):
+            If true, indicates that evaluation was generated by Hugging Face (vs. self-reported).
+            If a user tries to push self-reported metric results with verified=True, the push
+            will be rejected.
+        dataset_config (`str`, *optional*):
+            Example: fr. The name of the dataset configuration used in `load_dataset()`.
+        dataset_split (`str`, *optional*):
+            Example: test. The name of the dataset split used in `load_dataset()`.
+        dataset_revision (`str`, *optional*):
+            Example: 5503434ddd753f426f4b38109466949a1217c2bb. The name of the dataset dataset revision
+            used in `load_dataset()`.
+
+    Returns:
+        `dict`: a metadata dict with the result from a model evaluated on a dataset.
+
+    Example:
+    >>> from huggingface_hub import metadata_eval_result
+    >>> metadata_eval_result(
+    ...         model_pretty_name="RoBERTa fine-tuned on ReactionGIF",
+    ...         task_pretty_name="Text Classification",
+    ...         task_id="text-classification",
+    ...         metrics_pretty_name="Accuracy",
+    ...         metrics_id="accuracy",
+    ...         metrics_value=0.2662102282047272,
+    ...         dataset_pretty_name="ReactionJPEG",
+    ...         dataset_id="julien-c/reactionjpeg",
+    ...         dataset_config="default",
+    ...         dataset_split="test",
+    ...     )
+    {
+        "model-index": [
+            {
+                "name": "RoBERTa fine-tuned on ReactionGIF",
+                "results": [
+                    {
+                        "task": {
+                            "type": "text-classification",
+                            "name": "Text Classification",
+                        },
+                        "dataset": {
+                            "name": "ReactionJPEG",
+                            "type": "julien-c/reactionjpeg",
+                            "config": "default",
+                            "split": "test",
+                        },
+                        "metrics": [
+                            {
+                                "type": "accuracy",
+                                "value": 0.2662102282047272,
+                                "name": "Accuracy",
+                                "verified": False,
+                            }
+                        ],
+                    }
+                ],
+            }
+        ]
+    }
+    """
     model_index = ModelIndex(
         name=model_pretty_name,
         results=[
@@ -96,10 +183,18 @@ def metadata_eval_result(
                         type=metrics_id,
                         name=metrics_pretty_name,
                         value=metrics_value,
+                        config=metrics_config,
+                        verified=metrics_verified,
                     ),
                 ],
                 task=SingleResultTask(type=task_id, name=task_pretty_name),
-                dataset=SingleResultDataset(name=dataset_pretty_name, type=dataset_id),
+                dataset=SingleResultDataset(
+                    name=dataset_pretty_name,
+                    type=dataset_id,
+                    config=dataset_config,
+                    split=dataset_split,
+                    revision=dataset_revision,
+                ),
             )
         ],
     )
diff --git a/src/huggingface_hub/repocard_types.py b/src/huggingface_hub/repocard_types.py
index 8dc358e5..87537d0e 100644
--- a/src/huggingface_hub/repocard_types.py
+++ b/src/huggingface_hub/repocard_types.py
@@ -1,5 +1,3 @@
-# Code generated by jtd-codegen for Python v0.3.1
-
 from dataclasses import dataclass, field
 from typing import Any, List, Optional
 
@@ -19,7 +17,7 @@ class ModelIndex:
 class SingleMetric:
     type: str
     """
-    Example: wer
+    Example: wer. Use metric id from https://hf.co/metrics
     """
 
     value: Any
@@ -27,8 +25,26 @@ class SingleMetric:
     Example: 20.0 or "20.0 ± 1.2"
     """
 
+    name: Optional[str] = None
+    """
+    Example: Test WER
+    """
+
+    config: Optional[str] = None
+    """
+    The name of the metric configuration used in `load_metric()`. Example: bleurt-large-512 in `load_metric("bleurt", "bleurt-large-512")`. \
+    See the `datasets` docs for more info: https://huggingface.co/docs/datasets/v2.1.0/en/loading#load-configurations
+    """
+
     args: Any = field(default=None)
-    name: Optional[str] = field(default=None)
+    """
+    The arguments passed during `Metric.compute()`. Example for `bleu`: max_order: 4
+    """
+
+    verified: Optional[bool] = False
+    """
+    If true, indicates that evaluation was generated by Hugging Face (vs. self-reported).
+    """
 
 
 @dataclass
@@ -55,8 +71,7 @@ class SingleResultDataset:
 
     name: str
     """
-    Example: Common Voice zh-CN Also encode config params into the name if
-    relevant.
+    Example: Common Voice (French). A pretty name for the dataset.
     """
 
     type: str
@@ -64,6 +79,21 @@ class SingleResultDataset:
     Example: common_voice. Use dataset id from https://hf.co/datasets
     """
 
+    config: Optional[str] = None
+    """
+    Example: fr. The name of the dataset configuration used in `load_dataset()`
+    """
+
+    split: Optional[str] = None
+    """
+    Example: test.
+    """
+
+    revision: Optional[str] = None
+    """
+    Example: 5503434ddd753f426f4b38109466949a1217c2bb.
+    """
+
     args: Any = None
     """
     Example: zh-CN
@@ -72,9 +102,9 @@ class SingleResultDataset:
 
 @dataclass
 class SingleResult:
-    metrics: "List[SingleMetric]"
     task: "SingleResultTask"
     dataset: "Optional[SingleResultDataset]"
+    metrics: "List[SingleMetric]"
     """
     This will switch to required at some point. in any case, we need them to
     link to PWC


Test patch:
diff --git a/tests/test_repocard.py b/tests/test_repocard.py
index f857e8be..075d309e 100644
--- a/tests/test_repocard.py
+++ b/tests/test_repocard.py
@@ -94,16 +94,20 @@ DUMMY_MODELCARD_EVAL_RESULT = """---
 model-index:
 - name: RoBERTa fine-tuned on ReactionGIF
   results:
-  - metrics:
-    - type: accuracy
-      value: 0.2662102282047272
-      name: Accuracy
-    task:
+  - task:
       type: text-classification
       name: Text Classification
     dataset:
       name: ReactionGIF
       type: julien-c/reactiongif
+      config: default
+      split: test
+    metrics:
+    - type: accuracy
+      value: 0.2662102282047272
+      name: Accuracy
+      config: default
+      verified: false
 ---
 """
 
@@ -171,8 +175,12 @@ class RepocardTest(unittest.TestCase):
             metrics_pretty_name="Accuracy",
             metrics_id="accuracy",
             metrics_value=0.2662102282047272,
+            metrics_config="default",
+            metrics_verified=False,
             dataset_pretty_name="ReactionGIF",
             dataset_id="julien-c/reactiongif",
+            dataset_config="default",
+            dataset_split="test",
         )
         filename = "eval_results.md"
         filepath = Path(REPOCARD_DIR) / filename
@@ -307,8 +315,12 @@ class RepocardUpdateTest(unittest.TestCase):
             metrics_pretty_name="Recall",
             metrics_id="recall",
             metrics_value=0.7762102282047272,
+            metrics_config="default",
+            metrics_verified=False,
             dataset_pretty_name="ReactionGIF",
             dataset_id="julien-c/reactiongif",
+            dataset_config="default",
+            dataset_split="test",
         )
 
         metadata_update(
@@ -332,8 +344,12 @@ class RepocardUpdateTest(unittest.TestCase):
             metrics_pretty_name="Accuracy",
             metrics_id="accuracy",
             metrics_value=0.2662102282047272,
+            metrics_config="default",
+            metrics_verified=False,
             dataset_pretty_name="ReactionJPEG",
             dataset_id="julien-c/reactionjpeg",
+            dataset_config="default",
+            dataset_split="test",
         )
 
         metadata_update(