haoyang commited on
Commit
16b1a0b
•
1 Parent(s): 4b33d2c

update model metadata

Browse files
scripts/create_request_file.py CHANGED
@@ -12,7 +12,7 @@ EVAL_REQUESTS_PATH = "eval-queue"
12
  QUEUE_REPO = "open-llm-leaderboard/requests"
13
 
14
  precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ")
15
- model_types = ("pretrained", "fine-tuned", "RL-tuned", "instruction-tuned")
16
  weight_types = ("Original", "Delta", "Adapter")
17
 
18
 
 
12
  QUEUE_REPO = "open-llm-leaderboard/requests"
13
 
14
  precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ")
15
+ model_types = ("open-source", "close-source")
16
  weight_types = ("Original", "Delta", "Adapter")
17
 
18
 
src/display/utils.py CHANGED
@@ -65,10 +65,8 @@ class ModelDetails:
65
 
66
 
67
  class ModelType(Enum):
68
- PT = ModelDetails(name="pretrained", symbol="🟢")
69
- FT = ModelDetails(name="fine-tuned", symbol="🔶")
70
- IFT = ModelDetails(name="instruction-tuned", symbol="â­•")
71
- RL = ModelDetails(name="RL-tuned", symbol="🟦")
72
  Unknown = ModelDetails(name="", symbol="?")
73
 
74
  def to_str(self, separator=" "):
@@ -76,14 +74,10 @@ class ModelType(Enum):
76
 
77
  @staticmethod
78
  def from_str(type):
79
- if "fine-tuned" in type or "🔶" in type:
80
- return ModelType.FT
81
- if "pretrained" in type or "🟢" in type:
82
- return ModelType.PT
83
- if "RL-tuned" in type or "🟦" in type:
84
- return ModelType.RL
85
- if "instruction-tuned" in type or "â­•" in type:
86
- return ModelType.IFT
87
  return ModelType.Unknown
88
 
89
  class WeightType(Enum):
 
65
 
66
 
67
  class ModelType(Enum):
68
+ OSM = ModelDetails(name="open-source", symbol="🟢")
69
+ CSM = ModelDetails(name="close-source", symbol="🔶")
 
 
70
  Unknown = ModelDetails(name="", symbol="?")
71
 
72
  def to_str(self, separator=" "):
 
74
 
75
  @staticmethod
76
  def from_str(type):
77
+ if "close-source" in type or "🔶" in type:
78
+ return ModelType.CSM
79
+ if "open-source" in type or "🟢" in type:
80
+ return ModelType.OSM
 
 
 
 
81
  return ModelType.Unknown
82
 
83
  class WeightType(Enum):
src/leaderboard/read_evals.py CHANGED
@@ -64,8 +64,6 @@ class EvalResult:
64
  architectures = getattr(model_config, "architectures", None)
65
  if architectures:
66
  architecture = ";".join(architectures)
67
- if model_type == ModelType.Unknown:
68
- model_type = ModelType.from_str(getattr(model_config, "model_type", "Unknown"))
69
 
70
  # Extract results available in this file (some results are split in several files)
71
  results = {}
 
64
  architectures = getattr(model_config, "architectures", None)
65
  if architectures:
66
  architecture = ";".join(architectures)
 
 
67
 
68
  # Extract results available in this file (some results are split in several files)
69
  results = {}