Redo metric

heq_task.py

@@ -3,6 +3,7 @@ import string
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.metrics import Metrics, MetricCategory
 from lighteval.metrics.utils import CorpusLevelMetric, MetricUseCase
+from aenum import extend_enum
 import numpy as np
 from lighteval.tasks.requests import Doc
 from Levenshtein import distance
@@ -81,6 +82,7 @@ heq_tlnls_metric = CorpusLevelMetric(
     corpus_level_fn=np.mean,
     sample_level_fn=heq_eval_fn
 )
+extend_enum(Metrics, 'heq_tlnls_metric', heq_tlnls_metric)
 
 def heq_prompt_fn(line, task_name: str = None):
     """Defines how to go from a dataset line to a doc object.
@@ -106,6 +108,6 @@ heq_task = LightevalTaskConfig(
     hf_subset="default",
     hf_avail_splits=["heq"],
     evaluation_splits=["heq"],
-    metric=[heq_tlnls_metric],
+    metric=['heq_tlnls_metric'],
     stop_sequence=['\n']
 )

main_backend_lighteval.py

@@ -13,16 +13,15 @@ def patched_init(self, *args, **kwargs):
     self.name = self.name.replace('.', '-')
 lighteval.models.endpoint_model.InferenceEndpointModelConfig.__init__ = patched_init
 
-import huggingface_hub
-orig_create_endpoint = huggingface_hub.create_inference_endpoint
-def new_create_endpoint(*args, **kwargs):
-    print('$$$$$$$$$$$$$$$$$ here 1')
-    if 'custom_image' in kwargs and kwargs['custom_image']['url'] == "ghcr.io/huggingface/text-generation-inference:1.1.0":
-        print('$$$$$$$$$$$$$ here 2')
-        kwargs['custom_image']['url'] = "registry.internal.huggingface.tech/api-inference/community/text-generation-inference:gemma-ie"
-    return orig_create_endpoint(*args, **kwargs)
-huggingface_hub.create_inference_endpoint = new_create_endpoint
-
+# import huggingface_hub
+# orig_create_endpoint = huggingface_hub.create_inference_endpoint
+# def new_create_endpoint(*args, **kwargs):
+#     print('$$$$$$$$$$$$$$$$$ here 1')
+#     if 'custom_image' in kwargs and kwargs['custom_image']['url'] == "ghcr.io/huggingface/text-generation-inference:1.1.0":
+#         print('$$$$$$$$$$$$$ here 2')
+#         kwargs['custom_image']['url'] = "registry.internal.huggingface.tech/api-inference/community/text-generation-inference:gemma-ie"
+#     return orig_create_endpoint(*args, **kwargs)
+# huggingface_hub.create_inference_endpoint = new_create_endpoint
 
 from huggingface_hub import snapshot_download