Muennighoff commited on
Commit
be5f904
1 Parent(s): 85a6939

Add skip models

Browse files
Files changed (1) hide show
  1. app.py +6 -0
app.py CHANGED
@@ -660,6 +660,11 @@ MODELS_TO_SKIP = {
660
  "cgldo/semanticClone",
661
  "Malmuk1/e5-large-v2_Sharded",
662
  "jncraton/gte-small-ct2-int8",
 
 
 
 
 
663
  }
664
 
665
  EXTERNAL_MODEL_RESULTS = {model: {k: {v: []} for k, v in TASK_TO_METRIC.items()} for model in EXTERNAL_MODELS}
@@ -803,6 +808,7 @@ def get_mteb_data(tasks=["Clustering"], langs=[], datasets=[], fillna=True, add_
803
  # ],
804
  # },
805
  # Use "get" instead of dict indexing to skip incompat metadata instead of erroring out
 
806
  if len(datasets) > 0:
807
  task_results = [sub_res for sub_res in meta["model-index"][0]["results"] if (sub_res.get("task", {}).get("type", "") in tasks) and any([x in sub_res.get("dataset", {}).get("name", "") for x in datasets])]
808
  elif langs:
 
660
  "cgldo/semanticClone",
661
  "Malmuk1/e5-large-v2_Sharded",
662
  "jncraton/gte-small-ct2-int8",
663
+ "Einas/einas_ashkar",
664
+ "gruber/e5-small-v2-ggml",
665
+ "jncraton/bge-small-en-ct2-int8",
666
+ "vectoriseai/bge-small-en",
667
+ "recipe/embeddings",
668
  }
669
 
670
  EXTERNAL_MODEL_RESULTS = {model: {k: {v: []} for k, v in TASK_TO_METRIC.items()} for model in EXTERNAL_MODELS}
 
808
  # ],
809
  # },
810
  # Use "get" instead of dict indexing to skip incompat metadata instead of erroring out
811
+ print("RUNNING", model)
812
  if len(datasets) > 0:
813
  task_results = [sub_res for sub_res in meta["model-index"][0]["results"] if (sub_res.get("task", {}).get("type", "") in tasks) and any([x in sub_res.get("dataset", {}).get("name", "") for x in datasets])]
814
  elif langs: