Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
Clean up
Browse files- src/about.py +8 -7
src/about.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1 |
from dataclasses import dataclass
|
2 |
from enum import Enum
|
3 |
|
|
|
4 |
@dataclass
|
5 |
class Task:
|
6 |
benchmark: str
|
@@ -65,7 +66,7 @@ class Tasks(Enum):
|
|
65 |
xlsum_ja_rougeLsum = Task("scores", "xlsum_ja_rougeLsum", "XL-Sum ROUGE-Lsum")
|
66 |
|
67 |
|
68 |
-
NUM_FEWSHOT = 0
|
69 |
# ---------------------------------------------------
|
70 |
|
71 |
# Your leaderboard name
|
@@ -79,7 +80,7 @@ On the __"LLM Benchmark"__ page, the question mark **"?"** refers to the paramet
|
|
79 |
"""
|
80 |
|
81 |
# Which evaluations are you running? how can people reproduce what you have?
|
82 |
-
LLM_BENCHMARKS_TEXT =
|
83 |
## How it works
|
84 |
📈 We evaluate Japanese Large Language Models on 16 tasks leveraging our evaluation tool [llm-jp-eval](https://github.com/llm-jp/llm-jp-eval), a unified framework to evaluate Japanese LLMs on various evaluation tasks.
|
85 |
|
@@ -101,7 +102,7 @@ LLM_BENCHMARKS_TEXT = f"""
|
|
101 |
|
102 |
* `NIILC`, NIILC Question Answering Dataset [Source](https://github.com/mynlp/niilc-qa) (License CC BY-SA 4.0)
|
103 |
|
104 |
-
* `JAQKET`, Japanese QA dataset on the subject of quizzes [Source](https://www.nlp.ecei.tohoku.ac.jp/projects/jaqket/) (License CC BY-SA 4.0 - Other licenses are required for corporate usage
|
105 |
|
106 |
**RC (Reading Comprehension)**
|
107 |
|
@@ -147,7 +148,7 @@ This task is supported by llm-jp-eval, but it is not included in the evaluation
|
|
147 |
|
148 |
* `MMLU`, Measuring Massive Multitask Language Understanding [Source](https://github.com/hendrycks/test) (License MIT License)
|
149 |
|
150 |
-
* `JMMLU`, Japanese Massive Multitask Language Understanding Benchmark [Source](https://github.com/nlp-waseda/JMMLU) (License CC BY-SA 4.0
|
151 |
|
152 |
**CG (Code Generation)**
|
153 |
|
@@ -214,12 +215,12 @@ CITATION_BUTTON_TEXT = r"""@misc{OJLL,
|
|
214 |
howpublished = "\url{https://huggingface.co/spaces/llm-jp/open-japanese-llm-leaderboard}"
|
215 |
}
|
216 |
@misc{llmjp2024llmjpcrossorganizationalprojectresearch,
|
217 |
-
title={LLM-jp: A Cross-organizational Project for the Research and Development of Fully Open Japanese LLMs},
|
218 |
author={LLM-jp and : and Akiko Aizawa and Eiji Aramaki and Bowen Chen and Fei Cheng and Hiroyuki Deguchi and Rintaro Enomoto and Kazuki Fujii and Kensuke Fukumoto and Takuya Fukushima and Namgi Han and Yuto Harada and Chikara Hashimoto and Tatsuya Hiraoka and Shohei Hisada and Sosuke Hosokawa and Lu Jie and Keisuke Kamata and Teruhito Kanazawa and Hiroki Kanezashi and Hiroshi Kataoka and Satoru Katsumata and Daisuke Kawahara and Seiya Kawano and Atsushi Keyaki and Keisuke Kiryu and Hirokazu Kiyomaru and Takashi Kodama and Takahiro Kubo and Yohei Kuga and Ryoma Kumon and Shuhei Kurita and Sadao Kurohashi and Conglong Li and Taiki Maekawa and Hiroshi Matsuda and Yusuke Miyao and Kentaro Mizuki and Sakae Mizuki and Yugo Murawaki and Ryo Nakamura and Taishi Nakamura and Kouta Nakayama and Tomoka Nakazato and Takuro Niitsuma and Jiro Nishitoba and Yusuke Oda and Hayato Ogawa and Takumi Okamoto and Naoaki Okazaki and Yohei Oseki and Shintaro Ozaki and Koki Ryu and Rafal Rzepka and Keisuke Sakaguchi and Shota Sasaki and Satoshi Sekine and Kohei Suda and Saku Sugawara and Issa Sugiura and Hiroaki Sugiyama and Hisami Suzuki and Jun Suzuki and Toyotaro Suzumura and Kensuke Tachibana and Yu Takagi and Kyosuke Takami and Koichi Takeda and Masashi Takeshita and Masahiro Tanaka and Kenjiro Taura and Arseny Tolmachev and Nobuhiro Ueda and Zhen Wan and Shuntaro Yada and Sakiko Yahata and Yuya Yamamoto and Yusuke Yamauchi and Hitomi Yanaka and Rio Yokota and Koichiro Yoshino},
|
219 |
year={2024},
|
220 |
eprint={2407.03963},
|
221 |
archivePrefix={arXiv},
|
222 |
primaryClass={cs.CL},
|
223 |
-
url={https://arxiv.org/abs/2407.03963},
|
224 |
}
|
225 |
-
"""
|
|
|
1 |
from dataclasses import dataclass
|
2 |
from enum import Enum
|
3 |
|
4 |
+
|
5 |
@dataclass
|
6 |
class Task:
|
7 |
benchmark: str
|
|
|
66 |
xlsum_ja_rougeLsum = Task("scores", "xlsum_ja_rougeLsum", "XL-Sum ROUGE-Lsum")
|
67 |
|
68 |
|
69 |
+
NUM_FEWSHOT = 0 # Change with your few shot
|
70 |
# ---------------------------------------------------
|
71 |
|
72 |
# Your leaderboard name
|
|
|
80 |
"""
|
81 |
|
82 |
# Which evaluations are you running? how can people reproduce what you have?
|
83 |
+
LLM_BENCHMARKS_TEXT = """
|
84 |
## How it works
|
85 |
📈 We evaluate Japanese Large Language Models on 16 tasks leveraging our evaluation tool [llm-jp-eval](https://github.com/llm-jp/llm-jp-eval), a unified framework to evaluate Japanese LLMs on various evaluation tasks.
|
86 |
|
|
|
102 |
|
103 |
* `NIILC`, NIILC Question Answering Dataset [Source](https://github.com/mynlp/niilc-qa) (License CC BY-SA 4.0)
|
104 |
|
105 |
+
* `JAQKET`, Japanese QA dataset on the subject of quizzes [Source](https://www.nlp.ecei.tohoku.ac.jp/projects/jaqket/) (License CC BY-SA 4.0 - Other licenses are required for corporate usage)
|
106 |
|
107 |
**RC (Reading Comprehension)**
|
108 |
|
|
|
148 |
|
149 |
* `MMLU`, Measuring Massive Multitask Language Understanding [Source](https://github.com/hendrycks/test) (License MIT License)
|
150 |
|
151 |
+
* `JMMLU`, Japanese Massive Multitask Language Understanding Benchmark [Source](https://github.com/nlp-waseda/JMMLU) (License CC BY-SA 4.0 (3 tasks under the CC BY-NC-ND 4.0 license)
|
152 |
|
153 |
**CG (Code Generation)**
|
154 |
|
|
|
215 |
howpublished = "\url{https://huggingface.co/spaces/llm-jp/open-japanese-llm-leaderboard}"
|
216 |
}
|
217 |
@misc{llmjp2024llmjpcrossorganizationalprojectresearch,
|
218 |
+
title={LLM-jp: A Cross-organizational Project for the Research and Development of Fully Open Japanese LLMs},
|
219 |
author={LLM-jp and : and Akiko Aizawa and Eiji Aramaki and Bowen Chen and Fei Cheng and Hiroyuki Deguchi and Rintaro Enomoto and Kazuki Fujii and Kensuke Fukumoto and Takuya Fukushima and Namgi Han and Yuto Harada and Chikara Hashimoto and Tatsuya Hiraoka and Shohei Hisada and Sosuke Hosokawa and Lu Jie and Keisuke Kamata and Teruhito Kanazawa and Hiroki Kanezashi and Hiroshi Kataoka and Satoru Katsumata and Daisuke Kawahara and Seiya Kawano and Atsushi Keyaki and Keisuke Kiryu and Hirokazu Kiyomaru and Takashi Kodama and Takahiro Kubo and Yohei Kuga and Ryoma Kumon and Shuhei Kurita and Sadao Kurohashi and Conglong Li and Taiki Maekawa and Hiroshi Matsuda and Yusuke Miyao and Kentaro Mizuki and Sakae Mizuki and Yugo Murawaki and Ryo Nakamura and Taishi Nakamura and Kouta Nakayama and Tomoka Nakazato and Takuro Niitsuma and Jiro Nishitoba and Yusuke Oda and Hayato Ogawa and Takumi Okamoto and Naoaki Okazaki and Yohei Oseki and Shintaro Ozaki and Koki Ryu and Rafal Rzepka and Keisuke Sakaguchi and Shota Sasaki and Satoshi Sekine and Kohei Suda and Saku Sugawara and Issa Sugiura and Hiroaki Sugiyama and Hisami Suzuki and Jun Suzuki and Toyotaro Suzumura and Kensuke Tachibana and Yu Takagi and Kyosuke Takami and Koichi Takeda and Masashi Takeshita and Masahiro Tanaka and Kenjiro Taura and Arseny Tolmachev and Nobuhiro Ueda and Zhen Wan and Shuntaro Yada and Sakiko Yahata and Yuya Yamamoto and Yusuke Yamauchi and Hitomi Yanaka and Rio Yokota and Koichiro Yoshino},
|
220 |
year={2024},
|
221 |
eprint={2407.03963},
|
222 |
archivePrefix={arXiv},
|
223 |
primaryClass={cs.CL},
|
224 |
+
url={https://arxiv.org/abs/2407.03963},
|
225 |
}
|
226 |
+
"""
|