Hugging Face's logo

Spaces:
tsteffek
/
de-arena
Sleeping

App Files Community
de-arena
File size: 5,252 Bytes 
da92625
 
 
37b3751
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bfb3ae7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
da92625
 
 
 
 
 
 
 
 
 
 
 
 
bfb3ae7
da92625
 
 
 
 
 
606c189
 
 
 
 
 
 
 
 
 
 
 
 
da92625
 
 
887dd92
4575ae2
 
 
da92625
 
 
 
 
 
 
 
 
 
 
da96aa6
 
 
 
 
 
da92625
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
606c189
 
 
 
 
da92625
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
from dataclasses import dataclass
from enum import Enum


# @dataclass
# class Ranking:
#     dimension: str
#     metric: str
#     col_name: str

# class Rankings(Enum):
#     # dimension_key in the json file, metric_key in the json file, name to display in the leaderboard 
#     rank0 = Ranking("overall", "Avg Score", "Overall")
#     rank1 = Ranking("math_algebra", "Avg Score", "Math (Algebra)")
#     rank2 = Ranking("math_geometry", "Avg Score", "Math (Geometry)")
#     rank3 = Ranking("math_prob", "Avg Score", "Math (Probability)")
#     rank4 = Ranking("reason_logical", "Avg Score", "Logical Reasoning")
#     rank5 = Ranking("reason_social", "Avg Score", "Social Reasoning")


@dataclass
class Domain:
    dimension: str
    metric: str
    col_name: str
    
    
class Domains(Enum):
    # dimension_key in the json file, metric_key in the json file, name to display in the leaderboard 
    dim0 = Domain("overall", "Avg Rank", "Overall")
    # dim1 = Task("math", "acc", "Math")
    # dim2 = Task("math_algebra", "acc", "Algebra")
    
    

@dataclass
class Task:
    benchmark: str
    metric: str
    col_name: str


# Select your tasks here
# ---------------------------------------------------
class Tasks(Enum):
    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
    task0 = Task("anli_r1", "acc", "ANLI")
    task1 = Task("logiqa", "acc_norm", "LogiQA")
    
NUM_FEWSHOT = 0 # Change with your few shot
# ---------------------------------------------------



# Your leaderboard name
TITLE = """<h1 align="center" id="space-title">Decentralized Arena Leaderboard</h1>"""

SUB_TITLE = """<h3 align="center" id="space-subtitle">Building Automated, Robust, and Transparent LLM Evaluation for Numerous Dimensions</h3>"""

EXTERNAL_LINKS = """
<h3 align="center" id="space-links">
    <a href="https://de-arena.maitrix.org/" target="_blank">Blog</a> |
    <a href="https://github.com/maitrix-org/de-arena" target="_blank">GitHub</a> |
    <a href="https://de-arena.maitrix.org/images/Heading.mp4" target="">Video</a> |
    <a href="https://maitrix.org/" target="_blank">@Maitrix.org</a> |
    <a href="https://www.llm360.ai/" target="_blank">@LLM360</a>
</h3>
"""

# What does your leaderboard evaluate?
INTRODUCTION_TEXT = """
**Decentralized Arena** automates and scales "Chatbot Arena" for LLM evaluation across various fine-grained dimensions 
(e.g., math – algebra, geometry, probability; logical reasoning, social reasoning, biology, chemistry, …). 
The evaluation is decentralized and democratic, with all LLMs participating in evaluating others. 
It achieves a 95\% correlation with Chatbot Arena's overall rankings, while being fully transparent and reproducible.
"""

# Which evaluations are you running? how can people reproduce what you have?
LLM_BENCHMARKS_TEXT = f"""
## How it works

## Reproducibility
To reproduce our results, here is the commands you can run:

"""

COMING_SOON_TEXT = """
# Coming soon
We are working on adding more tasks to the leaderboard. Stay tuned!
"""


EVALUATION_QUEUE_TEXT = """
## Some good practices before submitting a model

### 1) Make sure you can load your model and tokenizer using AutoClasses:
```python
from transformers import AutoConfig, AutoModel, AutoTokenizer
config = AutoConfig.from_pretrained("your model name", revision=revision)
model = AutoModel.from_pretrained("your model name", revision=revision)
tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
```
If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.

Note: make sure your model is public!
Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!

### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!

### 3) Make sure your model has an open license!
This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗

### 4) Fill up your model card
When we add extra information about models to the leaderboard, it will be automatically taken from the model card

## In case of model failure
If your model is displayed in the `FAILED` category, its execution stopped.
Make sure you have followed the above steps first.
If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
"""

CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
CITATION_BUTTON_TEXT = r"""
@misc{decentralized2024,
  title={Decentralized Arena via Collective LLM Intelligence: Building Automated, Robust, and Transparent LLM Evaluation for Numerous Dimensions},
  author={Yanbin Yin, Zhen Wang, Kun Zhou, Xiangdong Zhang, Shibo Hao, Yi Gu, Jieyuan Liu, Somanshu Singla, Tianyang Liu, Eric P. Xing, Zhengzhong Liu, Haojian Jin, Zhiting Hu},
  year=2024
}
"""