adding lm_eval class

app.py

@@ -0,0 +1,52 @@
+import evaluate
+from evaluate.utils import infer_gradio_input_types, parse_gradio_data, json_to_string_type, parse_readme
+from pathlib import Path
+import sys
+
+def launch_gradio_widget(metric):
+    """Launches metric widget with Gradio."""
+    try:
+        import gradio as gr
+    except ImportError as error:
+        raise ImportError("To create a metric widget with Gradio, make sure gradio is installed.") from error
+
+    local_path = Path(sys.path[0])
+    
+    if isinstance(metric.features, list):
+        (feature_names, feature_types) = zip(*metric.features[0].items())
+    else:
+        (feature_names, feature_types) = zip(*metric.features.items())
+    
+    gradio_input_types = infer_gradio_input_types(feature_types)
+    
+    def compute(data):
+        return metric._compute(
+            model='gpt2',
+            tasks='wikitext', 
+            **parse_gradio_data(data, gradio_input_types)
+        )
+    
+    iface = gr.Interface(
+        fn=compute,
+        inputs=gr.Dataframe(
+            headers=feature_names,
+            col_count=len(feature_names),
+            row_count=1,
+            datatype=json_to_string_type(gradio_input_types),
+        ),
+        outputs=gr.Textbox(label=metric.name),
+        description=(
+            metric.info.description + "\nThis metric is computed using the 'gpt2' model on the 'wikitext' task.\n"
+            "Ensure your input is appropriate for the selected task. "
+            "If this is a text-based metric, wrap your input in double quotes."
+            " Alternatively, you can use a JSON-formatted list as input."
+        ),
+        title=f"Metric: {metric.name}",
+        article=parse_readme(local_path / "README.md"),
+    )
+    
+    iface.launch()
+
+module = evaluate.load("d-matrix/dmx_lm_eval")
+
+launch_gradio_widget(module)

dmx_lm_eval.py

@@ -0,0 +1,96 @@
+import evaluate
+import lm_eval
+from typing import Union, List, Optional
+from dmx.compressor.dmx import config_rules, DmxModel
+import datasets
+
+_DESCRIPTION = """
+Evaluation function using lm-eval with d-Matrix integration.
+This function allows for the evaluation of language models across various tasks, 
+with the option to use d-Matrix compressed models.
+"""
+
+_KWARGS_DESCRIPTION = """
+Args:
+    model (str): The name or path of the model to evaluate.
+    tasks (Union[str, List[str]]): The task or list of tasks to evaluate on.
+    dmx_config (Optional[str]): Configuration string for d-Matrix transformations, defaults to None.
+    num_fewshot (Optional[int]): Number of examples in few-shot context, defaults to None.
+    batch_size (Optional[Union[int, str]]): Batch size for model, defaults to None.
+    max_batch_size (Optional[int]): Maximum batch size to try with automatic batch size detection, defaults to None.
+    limit (Optional[Union[int, float]]): Limit the number of examples per task, defaults to None.
+    revision (str): Model revision to use, defaults to 'main'.
+    trust_remote_code (bool): Whether to trust remote code, defaults to False.
+    log_samples (bool): If True, logs all model outputs and documents, defaults to True.
+    verbosity (str): Logging verbosity level, defaults to 'CRITICAL'.
+    **kwargs: Additional keyword arguments to pass to `lm_eval.evaluate`.
+
+Returns:
+    dict: A dictionary containing the evaluation results.
+"""
+
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class  DmxMetric(evaluate.Metric):
+    def _info(self):
+        return evaluate.MetricInfo(
+            module_type="metric",
+            description=_DESCRIPTION,
+            citation="",
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=datasets.Features(
+                {
+                    "references": datasets.Value("string"),
+                }
+            ),
+            reference_urls=["https://github.com/EleutherAI/lm-evaluation-harness"],
+        )
+
+    def _compute(
+        self,
+        model: str,
+        tasks: Union[str, List[str]],
+        dmx_config: Optional[str] = None,
+        num_fewshot: Optional[int] = None,
+        batch_size: Optional[Union[int, str]] = None,
+        max_batch_size: Optional[int] = None,
+        limit: Optional[Union[int, float]] = None,
+        revision: str = "main",
+        trust_remote_code: bool = False, 
+        log_samples: bool = True,
+        verbosity: str = "INFO",
+        **kwargs
+    ):
+        """
+        Evaluate a model on multiple tasks and metrics using lm-eval with optional d-Matrix integration.
+        """
+        model_args = f"pretrained={model},revision={revision},trust_remote_code={str(trust_remote_code)}"
+
+        lm = lm_eval.api.registry.get_model("hf").create_from_arg_string(
+            model_args,
+            {
+                "batch_size": batch_size,
+                "max_batch_size": max_batch_size,
+            }
+        )
+        
+        if dmx_config:
+            lm._model = DmxModel.from_torch(lm._model)
+            lm._model.transform(lm._model.dmx_config, *eval(f"config_rules.{dmx_config}"))
+        
+        task_dict = lm_eval.tasks.get_task_dict(tasks if isinstance(tasks, list) else [tasks])
+    
+        for task in task_dict.values():
+            if num_fewshot is not None:
+                task.set_config(key="num_fewshot", value=num_fewshot)
+        
+        eval_params = {
+            'lm': lm,
+            'task_dict': task_dict,
+            'limit': limit,
+            'log_samples': log_samples,
+            'verbosity': verbosity,
+            **kwargs
+        }
+        
+        results = lm_eval.evaluate(**eval_params)
+        return results.get('results', {})