hysts HF staff commited on
Commit
aed7c1c
·
1 Parent(s): 57cc619
Files changed (1) hide show
  1. src/about.py +8 -7
src/about.py CHANGED
@@ -1,6 +1,7 @@
1
  from dataclasses import dataclass
2
  from enum import Enum
3
 
 
4
  @dataclass
5
  class Task:
6
  benchmark: str
@@ -65,7 +66,7 @@ class Tasks(Enum):
65
  xlsum_ja_rougeLsum = Task("scores", "xlsum_ja_rougeLsum", "XL-Sum ROUGE-Lsum")
66
 
67
 
68
- NUM_FEWSHOT = 0 # Change with your few shot
69
  # ---------------------------------------------------
70
 
71
  # Your leaderboard name
@@ -79,7 +80,7 @@ On the __"LLM Benchmark"__ page, the question mark **"?"** refers to the paramet
79
  """
80
 
81
  # Which evaluations are you running? how can people reproduce what you have?
82
- LLM_BENCHMARKS_TEXT = f"""
83
  ## How it works
84
  📈 We evaluate Japanese Large Language Models on 16 tasks leveraging our evaluation tool [llm-jp-eval](https://github.com/llm-jp/llm-jp-eval), a unified framework to evaluate Japanese LLMs on various evaluation tasks.
85
 
@@ -101,7 +102,7 @@ LLM_BENCHMARKS_TEXT = f"""
101
 
102
  * `NIILC`, NIILC Question Answering Dataset [Source](https://github.com/mynlp/niilc-qa) (License CC BY-SA 4.0)
103
 
104
- * `JAQKET`, Japanese QA dataset on the subject of quizzes [Source](https://www.nlp.ecei.tohoku.ac.jp/projects/jaqket/) (License CC BY-SA 4.0 - Other licenses are required for corporate usage
105
 
106
  **RC (Reading Comprehension)**
107
 
@@ -147,7 +148,7 @@ This task is supported by llm-jp-eval, but it is not included in the evaluation
147
 
148
  * `MMLU`, Measuring Massive Multitask Language Understanding [Source](https://github.com/hendrycks/test) (License MIT License)
149
 
150
- * `JMMLU`, Japanese Massive Multitask Language Understanding Benchmark [Source](https://github.com/nlp-waseda/JMMLU) (License CC BY-SA 4.03 tasks under the CC BY-NC-ND 4.0 license)
151
 
152
  **CG (Code Generation)**
153
 
@@ -214,12 +215,12 @@ CITATION_BUTTON_TEXT = r"""@misc{OJLL,
214
  howpublished = "\url{https://huggingface.co/spaces/llm-jp/open-japanese-llm-leaderboard}"
215
  }
216
  @misc{llmjp2024llmjpcrossorganizationalprojectresearch,
217
- title={LLM-jp: A Cross-organizational Project for the Research and Development of Fully Open Japanese LLMs},
218
  author={LLM-jp and : and Akiko Aizawa and Eiji Aramaki and Bowen Chen and Fei Cheng and Hiroyuki Deguchi and Rintaro Enomoto and Kazuki Fujii and Kensuke Fukumoto and Takuya Fukushima and Namgi Han and Yuto Harada and Chikara Hashimoto and Tatsuya Hiraoka and Shohei Hisada and Sosuke Hosokawa and Lu Jie and Keisuke Kamata and Teruhito Kanazawa and Hiroki Kanezashi and Hiroshi Kataoka and Satoru Katsumata and Daisuke Kawahara and Seiya Kawano and Atsushi Keyaki and Keisuke Kiryu and Hirokazu Kiyomaru and Takashi Kodama and Takahiro Kubo and Yohei Kuga and Ryoma Kumon and Shuhei Kurita and Sadao Kurohashi and Conglong Li and Taiki Maekawa and Hiroshi Matsuda and Yusuke Miyao and Kentaro Mizuki and Sakae Mizuki and Yugo Murawaki and Ryo Nakamura and Taishi Nakamura and Kouta Nakayama and Tomoka Nakazato and Takuro Niitsuma and Jiro Nishitoba and Yusuke Oda and Hayato Ogawa and Takumi Okamoto and Naoaki Okazaki and Yohei Oseki and Shintaro Ozaki and Koki Ryu and Rafal Rzepka and Keisuke Sakaguchi and Shota Sasaki and Satoshi Sekine and Kohei Suda and Saku Sugawara and Issa Sugiura and Hiroaki Sugiyama and Hisami Suzuki and Jun Suzuki and Toyotaro Suzumura and Kensuke Tachibana and Yu Takagi and Kyosuke Takami and Koichi Takeda and Masashi Takeshita and Masahiro Tanaka and Kenjiro Taura and Arseny Tolmachev and Nobuhiro Ueda and Zhen Wan and Shuntaro Yada and Sakiko Yahata and Yuya Yamamoto and Yusuke Yamauchi and Hitomi Yanaka and Rio Yokota and Koichiro Yoshino},
219
  year={2024},
220
  eprint={2407.03963},
221
  archivePrefix={arXiv},
222
  primaryClass={cs.CL},
223
- url={https://arxiv.org/abs/2407.03963},
224
  }
225
- """
 
1
  from dataclasses import dataclass
2
  from enum import Enum
3
 
4
+
5
  @dataclass
6
  class Task:
7
  benchmark: str
 
66
  xlsum_ja_rougeLsum = Task("scores", "xlsum_ja_rougeLsum", "XL-Sum ROUGE-Lsum")
67
 
68
 
69
+ NUM_FEWSHOT = 0 # Change with your few shot
70
  # ---------------------------------------------------
71
 
72
  # Your leaderboard name
 
80
  """
81
 
82
  # Which evaluations are you running? how can people reproduce what you have?
83
+ LLM_BENCHMARKS_TEXT = """
84
  ## How it works
85
  📈 We evaluate Japanese Large Language Models on 16 tasks leveraging our evaluation tool [llm-jp-eval](https://github.com/llm-jp/llm-jp-eval), a unified framework to evaluate Japanese LLMs on various evaluation tasks.
86
 
 
102
 
103
  * `NIILC`, NIILC Question Answering Dataset [Source](https://github.com/mynlp/niilc-qa) (License CC BY-SA 4.0)
104
 
105
+ * `JAQKET`, Japanese QA dataset on the subject of quizzes [Source](https://www.nlp.ecei.tohoku.ac.jp/projects/jaqket/) (License CC BY-SA 4.0 - Other licenses are required for corporate usage)
106
 
107
  **RC (Reading Comprehension)**
108
 
 
148
 
149
  * `MMLU`, Measuring Massive Multitask Language Understanding [Source](https://github.com/hendrycks/test) (License MIT License)
150
 
151
+ * `JMMLU`, Japanese Massive Multitask Language Understanding Benchmark [Source](https://github.com/nlp-waseda/JMMLU) (License CC BY-SA 4.0 (3 tasks under the CC BY-NC-ND 4.0 license)
152
 
153
  **CG (Code Generation)**
154
 
 
215
  howpublished = "\url{https://huggingface.co/spaces/llm-jp/open-japanese-llm-leaderboard}"
216
  }
217
  @misc{llmjp2024llmjpcrossorganizationalprojectresearch,
218
+ title={LLM-jp: A Cross-organizational Project for the Research and Development of Fully Open Japanese LLMs},
219
  author={LLM-jp and : and Akiko Aizawa and Eiji Aramaki and Bowen Chen and Fei Cheng and Hiroyuki Deguchi and Rintaro Enomoto and Kazuki Fujii and Kensuke Fukumoto and Takuya Fukushima and Namgi Han and Yuto Harada and Chikara Hashimoto and Tatsuya Hiraoka and Shohei Hisada and Sosuke Hosokawa and Lu Jie and Keisuke Kamata and Teruhito Kanazawa and Hiroki Kanezashi and Hiroshi Kataoka and Satoru Katsumata and Daisuke Kawahara and Seiya Kawano and Atsushi Keyaki and Keisuke Kiryu and Hirokazu Kiyomaru and Takashi Kodama and Takahiro Kubo and Yohei Kuga and Ryoma Kumon and Shuhei Kurita and Sadao Kurohashi and Conglong Li and Taiki Maekawa and Hiroshi Matsuda and Yusuke Miyao and Kentaro Mizuki and Sakae Mizuki and Yugo Murawaki and Ryo Nakamura and Taishi Nakamura and Kouta Nakayama and Tomoka Nakazato and Takuro Niitsuma and Jiro Nishitoba and Yusuke Oda and Hayato Ogawa and Takumi Okamoto and Naoaki Okazaki and Yohei Oseki and Shintaro Ozaki and Koki Ryu and Rafal Rzepka and Keisuke Sakaguchi and Shota Sasaki and Satoshi Sekine and Kohei Suda and Saku Sugawara and Issa Sugiura and Hiroaki Sugiyama and Hisami Suzuki and Jun Suzuki and Toyotaro Suzumura and Kensuke Tachibana and Yu Takagi and Kyosuke Takami and Koichi Takeda and Masashi Takeshita and Masahiro Tanaka and Kenjiro Taura and Arseny Tolmachev and Nobuhiro Ueda and Zhen Wan and Shuntaro Yada and Sakiko Yahata and Yuya Yamamoto and Yusuke Yamauchi and Hitomi Yanaka and Rio Yokota and Koichiro Yoshino},
220
  year={2024},
221
  eprint={2407.03963},
222
  archivePrefix={arXiv},
223
  primaryClass={cs.CL},
224
+ url={https://arxiv.org/abs/2407.03963},
225
  }
226
+ """