Merge branch 'main' into pr/15

README.md

@@ -15,20 +15,71 @@ tags:
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
-## Local development
+# Contributing to Open-MOE-LLM-Leaderboard
 
-Create a virtual environment and install the dependencies:
+Thank you for your interest in contributing to the Open-MOE-LLM-Leaderboard project! We welcome contributions from everyone. Below you'll find guidance on how to set up your development environment, understand our architecture, and contribute effectively. If you have any questions or wish to discuss your contributions, please reach out to Yao Fu via email at [Y.Fu@ed.ac.uk](mailto:y.fu@ed.ac.uk).
+
+## What We're Looking For in Contributions
+
+We are looking for contributions in several key areas to enhance the Open-MOE-LLM-Leaderboard project:
+
+1. **General Bug Fixes/Reports**: We welcome reports of any bugs found in the frontend interface or backend, as well as fixes for these issues.
+
+2. **Adding New Tasks (Benchmark Datasets)**: If you have ideas for new benchmark datasets that could be added, your contributions would be greatly appreciated.
+
+3. **Supporting New Inference Frameworks**: Expanding our project to support new inference frameworks is crucial for our growth. If you can contribute in this area, please reach out.
+
+4. **Testing More Models**: To make our leaderboard as comprehensive as possible, we need to test a wide range of models. Contributions in this area are highly valuable.
+
+Documentation is currently of lower priority, but if you have thoughts or suggestions, please feel free to raise them.
+
+Your contributions are crucial to the success and improvement of the Open-MOE-LLM-Leaderboard project. We look forward to collaborating with you.
+
+
+## Development Setup
+
+To start contributing, set up your development environment as follows:
 
 ```bash
-conda create -n <env_name> python=3.10
-conda activate <env_name>
+conda create -n leaderboard python=3.10
+conda activate leaderboard
 pip install -r requirements.txt
+pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ moe-infinity
+pip install pydantic==2.6.4 # Resolves a dependency conflict with moe-infinity
+python -m spacy download en # Required for selfcheckgpt
+```
+
+## Architecture Overview
+
+The Open-MOE-LLM-Leaderboard project uses the following architecture:
+
+- **User Interface (Gradio)** ->upload-> **HuggingFace Dataset (Request)** ->download-> **Backend GPU Server** ->upload-> **HuggingFace Dataset (Result)** ->download-> **User Interface (Gradio)**
+
+In brief:
+1. Users submit model benchmarking requests through the Gradio interface ([app.py](./app.py)). These requests are then recorded in a HuggingFace dataset ([sparse-generative-ai/requests](https://huggingface.co/datasets/sparse-generative-ai/requests)).
+2. The backend ([backend-cli.py](./backend-cli.py)), running on a GPU server, processes these requests, performs the benchmarking tasks, and uploads the results to another HuggingFace dataset ([sparse-generative-ai/results](https://huggingface.co/datasets/sparse-generative-ai/results)).
+3. Finally, the Gradio interface retrieves and displays these results to the users.
+
+## Running the Gradio Interface
+
+To launch the Gradio interface, execute:
+
+```bash
+python app.py
 ```
 
-**Follow the instructions in Dockerfile to install other necessary dependencies.**
+Then, open your browser and navigate to http://127.0.0.1:7860.
 
-Start the backend server in debug mode:
+## Running the Backend
+
+To start the backend process, use:
 
 ```bash
 python backend-cli.py --debug
-```
+```
+
+For additional details, please consult the [backend-cli.py](./backend-cli.py) script.
+
+---
+
+We look forward to your contributions and are here to help guide you through the process. Thank you for supporting the Open-MOE-LLM-Leaderboard project!

requirements.txt

@@ -18,7 +18,7 @@ tqdm
 wandb
 transformers>=4.36.0
 tokenizers>=0.15.0
-lm_eval[ifeval] @ git+https://github.com/EleutherAI/lm-evaluation-harness.git
+lm_eval[ifeval] @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@v0.4.2
 accelerate
 sentencepiece
 langdetect

src/backend/hflm_with_measurement.py

@@ -68,6 +68,226 @@ class HFLMWithMeasurement(HFLM):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+    def _loglikelihood_tokens(
+        self,
+        requests: List[Tuple[Tuple[str, str], List[int], List[int]]],
+        disable_tqdm: bool = False,
+        override_bs: int = None,
+    ) -> List[Tuple[float, bool]]:
+        # TODO: implement some kind of efficient-request-middleware that lumps together requests with the same context
+        res = []
+
+        def _collate(req: Tuple[Tuple[str, str], List[int], List[int]]):
+            """Defines the key for the sorted method"""
+            # the negative sign on len(toks) sorts descending - this has a few advantages:
+            # - time estimates will always be over not underestimates, which is more useful for planning
+            # - to know the size of a batch when going through the list, you know the first one is always the batch
+            #   padded context length. this is useful to simplify the batching logic and more importantly to make
+            #   automatic adaptive batches much much easier to implement
+            # - any OOMs will happen right away rather than near the end
+
+            toks = req[1] + req[2]
+            return -len(toks), tuple(toks)
+
+        def _lookup_one_token_cont(req: Tuple[Tuple[str, str], List[int], List[int]]):
+            """Defines the key to group and lookup one-token continuations"""
+            # Use with group_by="contexts" (optional)"
+            # allows for the creation of a lookup, so we can reuse logits in case of one-token continuations.
+            # speeds up some multiple-choice tasks proportionally to the number of choices.
+            # groups requests by context+continuation[:-1] and infer on one request/group.
+            return req[-2] + req[-1][:-1]
+
+        re_ord = Collator(
+            requests,
+            sort_fn=_collate,
+            group_by="contexts"
+            if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM
+            and self.logits_cache
+            else None,
+            group_fn=_lookup_one_token_cont,
+        )
+
+        # automatic (variable) batch size detection for vectorization
+        # pull longest context sample from request
+        n_reordered_requests = len(re_ord)
+        batch_size = (
+            self.batch_size
+            if self.batch_size != "auto"
+            else override_bs
+            if override_bs is not None
+            else 0
+        )
+        batch_fn = (
+            self._batch_scheduler
+            if self.batch_size == "auto"
+            and n_reordered_requests > 0
+            and not override_bs
+            else None
+        )
+
+        chunks = re_ord.get_batched(n=batch_size, batch_fn=batch_fn)
+        pbar = tqdm(
+            total=len(requests),
+            disable=(disable_tqdm or (self.rank != 0)),
+            desc="Running loglikelihood requests",
+        )
+        for chunk in chunks:
+            inps = []
+            cont_toks_list = []
+            inplens = []
+
+            conts = []
+            encoder_attns = []
+
+            padding_len_inp = None
+            padding_len_cont = None
+            # because vectorizing is annoying, we first convert each (context, continuation) pair to padded
+            # tensors, then we pack them together into a batch, call the model, and then pick it all apart
+            # again because vectorizing is annoying
+
+            for _, context_enc, continuation_enc in chunk:
+                # sanity check
+                assert len(context_enc) > 0
+                assert len(continuation_enc) > 0
+                assert len(continuation_enc) <= self.max_length
+
+                # how this all works (illustrated on a causal decoder-only setup):
+                #          CTX      CONT
+                # inp    0 1 2 3|4 5 6 7 8 9   <- last token is deleted by inp[:, :-1]
+                # model  \               \
+                # logits   1 2 3|4 5 6 7 8 9   <- the ctx half gets tossed out by the
+                # cont_toks      4 5 6 7 8 9      [:, -len(continuation_enc):, :self.vocab_size] slice
+
+                # when too long to fit in context, truncate from the left
+                if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM:
+                    inp = torch.tensor(
+                        (context_enc + continuation_enc)[-(self.max_length + 1) :][:-1],
+                        dtype=torch.long,
+                        device=self.device,
+                    )
+                    (inplen,) = inp.shape
+                elif self.AUTO_MODEL_CLASS == transformers.AutoModelForSeq2SeqLM:
+                    inp = torch.tensor(
+                        (context_enc)[-self.max_length :],
+                        dtype=torch.long,
+                        device=self.device,
+                    )
+                    (inplen,) = inp.shape
+
+                    # build encoder attn masks
+                    encoder_attns.append(torch.ones_like(inp))
+
+                    cont = torch.tensor(
+                        (continuation_enc)[-self.max_length :],
+                        # TODO: left-shift these?
+                        # TODO: our code assumes we never end up truncating conts for either model type
+                        dtype=torch.long,
+                        device=self.device,
+                    )
+                    (contlen,) = cont.shape
+
+                    conts.append(cont)
+
+                    padding_len_cont = (
+                        max(padding_len_cont, contlen)
+                        if padding_len_cont is not None
+                        else contlen
+                    )
+
+                padding_len_inp = (
+                    max(padding_len_inp, inplen)
+                    if padding_len_inp is not None
+                    else inplen
+                )
+
+                inps.append(inp)  # [1, inp_length]
+                cont_toks_list.append(continuation_enc)
+                inplens.append(inplen)
+
+            # create encoder attn mask and batched conts, if seq2seq
+            call_kwargs = {}
+            if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM:
+                batched_inps = pad_and_concat(
+                    padding_len_inp, inps, padding_side="right"
+                )  # [batch, padding_len_inp]
+            elif self.AUTO_MODEL_CLASS == transformers.AutoModelForSeq2SeqLM:
+                # TODO: left-pad encoder inps and mask?
+                batched_inps = pad_and_concat(
+                    padding_len_inp, inps
+                )  # [batch, padding_len_inp]
+                batched_conts = pad_and_concat(
+                    padding_len_cont, conts
+                )  # [batch, padding_len_cont]
+                batched_encoder_mask = pad_and_concat(
+                    padding_len_inp, encoder_attns
+                )  # [batch, padding_len_inp]
+                call_kwargs = {
+                    "attn_mask": batched_encoder_mask,
+                    "labels": batched_conts,
+                }
+
+            start = time()
+            intermediate_res = self._model_call(batched_inps, **call_kwargs)
+            end = time()
+            multi_logits = F.log_softmax(
+                intermediate_res , dim=-1
+            )  # [batch, padding_length (inp or cont), vocab]
+            per_sample_time = (end - start) / len(multi_logits)
+
+            for (request_str, ctx_tokens, _), logits, inplen, cont_toks in zip(
+                chunk, multi_logits, inplens, cont_toks_list
+            ):
+                # Slice to original seq length
+                contlen = len(cont_toks)
+                # take only logits in the continuation
+                # (discard context toks if decoder-only ; discard right-padding)
+                # also discards + checks for "virtual tokens" in the causal LM's input window
+                # from prompt/prefix tuning tokens, if applicable
+                ctx_len = (
+                    inplen + (logits.shape[0] - padding_len_inp)
+                    if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM
+                    else None
+                )
+                logits = self._select_cont_toks(logits, contlen=contlen, inplen=ctx_len)
+                logits = logits.unsqueeze(0)  # [1, seq, vocab]
+
+                # Check if per-token argmax is exactly equal to continuation
+                greedy_tokens = logits.argmax(dim=-1)
+
+                # check for one-token continuation cache hits.
+                # noop in case group_by != "contexts" or no cache hit and returns the
+                # original args. Otherwise, expands the logits batch dimension and yields each
+                # batch along with matching continuation tokens and prompt strings.
+                # logits -> [1, seq, vocab]
+                for request_str, cont_toks, logits in re_ord.get_cache(
+                    req_str=request_str,
+                    cxt_toks=ctx_tokens,
+                    cont_toks=cont_toks,
+                    logits=logits,
+                ):
+                    cont_toks = torch.tensor(
+                        cont_toks, dtype=torch.long, device=self.device
+                    ).unsqueeze(0)  # [1, seq]
+                    max_equal = (greedy_tokens == cont_toks).all()
+
+                    # Obtain log-probs at the corresponding continuation token indices
+                    # last_token_slice = logits[:, -1, :].squeeze(0).tolist()
+                    logits = torch.gather(logits, 2, cont_toks.unsqueeze(-1)).squeeze(
+                        -1
+                    )  # [1, seq]
+
+                    # Answer: (log prob, is-exact-match)
+                    answer = (float(logits.sum()), bool(max_equal))
+
+                    res.append((answer, per_sample_time, 0, 0))
+
+                    self.cache_hook.add_partial("loglikelihood", request_str, answer)
+                    pbar.update(1)
+
+        pbar.close()
+
+        return re_ord.get_original(res)
+    
     def _model_generate(self, context, max_length, stop, **generation_kwargs):
         # temperature = 0.0 if not set
         # if do_sample is false and temp==0.0:

src/backend/run_eval_suite.py

@@ -1,13 +1,57 @@
 from lm_eval import evaluator
 from lm_eval.tasks import TaskManager
+from lm_eval.api.metrics import mean
+from lm_eval.api.task import ConfigurableTask
 
 from src.backend.manage_requests import EvalRequest
 
-from src.backend.tasks.xsum.task import XSum
-from src.backend.tasks.xsum.task_v2 import XSumv2
 
-from src.backend.tasks.cnndm.task import CNNDM
-from src.backend.tasks.cnndm.task_v2 import CNNDMv2
+orig_process_results = ConfigurableTask.process_results
+orig_aggregation = ConfigurableTask.aggregation
+orig_higher_is_better = ConfigurableTask.higher_is_better
+
+def process_results_decorator(func):
+    def wrapper(self, doc, results, *args, **kwargs):
+        processed_results = [r[0] for r in results]
+        
+        end_to_end_time = sum([r[1] for r in results]) / len(results)
+        prefilling_time = sum([r[2] for r in results]) / len(results)
+        decoding_throughput = sum([r[3] for r in results]) / len(results)
+        # print(f"end_to_end_time: {end_to_end_time}, prefilling_time: {prefilling_time}, decoding_throughput: {decoding_throughput}")
+
+        result_dict = func(self, doc, processed_results, *args, **kwargs)
+        result_dict["end_to_end_time"] = end_to_end_time
+        result_dict["prefilling_time"] = prefilling_time
+        result_dict["decoding_throughput"] = decoding_throughput
+        return result_dict
+    return wrapper
+ConfigurableTask.process_results = process_results_decorator(orig_process_results)
+
+def aggregation_decorator(func):
+    def wrapper(self, *args, **kwargs):
+        aggregation_list = func(self, *args, **kwargs)
+        aggregation_list["end_to_end_time"] = mean
+        aggregation_list["prefilling_time"] = mean
+        aggregation_list["decoding_throughput"] = mean
+        return aggregation_list
+    return wrapper
+ConfigurableTask.aggregation = aggregation_decorator(orig_aggregation)
+
+def higher_is_better_decorator(func):
+    def wrapper(self, *args, **kwargs):
+        higher_is_better_dict = func(self, *args, **kwargs)
+        higher_is_better_dict["end_to_end_time"] = False
+        higher_is_better_dict["prefilling_time"] = False
+        higher_is_better_dict["decoding_throughput"] = True
+        return higher_is_better_dict
+    return wrapper
+ConfigurableTask.higher_is_better = higher_is_better_decorator(orig_higher_is_better)
+
+# from src.backend.tasks.xsum.task import XSum
+# from src.backend.tasks.xsum.task_v2 import XSumv2
+
+# from src.backend.tasks.cnndm.task import CNNDM
+# from src.backend.tasks.cnndm.task_v2 import CNNDMv2
 
 from src.backend.tasks.selfcheckgpt.task import SelfCheckGPT
 

src/backend/tasks/measurement_task_utils.py

@@ -12,7 +12,7 @@ def process_results_decorator(func):
         end_to_end_time = sum([r[1] for r in results]) / len(results)
         prefilling_time = sum([r[2] for r in results]) / len(results)
         decoding_throughput = sum([r[3] for r in results]) / len(results)
-        print(f"end_to_end_time: {end_to_end_time}, prefilling_time: {prefilling_time}, decoding_throughput: {decoding_throughput}")
+        # print(f"end_to_end_time: {end_to_end_time}, prefilling_time: {prefilling_time}, decoding_throughput: {decoding_throughput}")
 
         # Now call the original process_results with the processed results
         result_dict = func(self, doc, processed_results, *args, **kwargs)

src/display/about.py

@@ -1,13 +1,25 @@
 from src.display.utils import ModelType
 
-TITLE = """<h1 align="center" id="space-title">MOE LLM GPU-Poor Leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">OPEN-MOE-LLM-LEADERBOARD</h1>"""
 
 INTRODUCTION_TEXT = """
-📐 The MOE LLM GPU-Poor Leaderboard aims to evaluate LLMs.
+The OPEN-MOE-LLM-LEADERBOARD is specifically designed to assess the performance and efficiency of various Mixture of Experts (MoE) Large Language Models (LLMs). This initiative, driven by the open-source community, aims to comprehensively evaluate these advanced MoE LLMs. We extend our gratitude to the Huggingface for the GPU community grant that supported the initial debugging process, and to [NetMind.AI](https://netmind.ai/home) for their generous GPU donation, which ensures the continuous operation of the Leaderboard.
 
+The OPEN-MOE-LLM-LEADERBOARD includes generation and multiple choice tasks to measure the performance and efficiency of MOE LLMs.
 
-"""
 
+Tasks:
+- **Generation Self-consistancy** -- [SelfCheckGPT](https://github.com/potsawee/selfcheckgpt)
+- **Multiple Choice Performance** -- [MMLU](https://arxiv.org/abs/2009.03300)
+
+Columns and Metrics:
+- Method: The MOE LLMs inference framework.
+- E2E(s): Average End to End generation time in seconds.
+- PRE(s): Prefilling Time of input prompt in seconds.
+- T/s: Tokens throughout per second.
+- Precision: The precison of used model.
+
+"""
 LLM_BENCHMARKS_TEXT = f"""
 
 """

src/display/utils.py

@@ -7,6 +7,11 @@ import pandas as pd
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
+E2Es = "E2E(s)" #"End-to-end time (s)"
+PREs = "PRE(s)" #"Prefilling time (s)"
+TS = "T/s" #Decoding throughput (tok/s)
+InFrame = "Method" #"Inference framework"
+MULTIPLE_CHOICEs = ["mmlu"]
 
 @dataclass
 class Task:
@@ -46,7 +51,7 @@ class Tasks(Enum):
 
     # # XXX include me back at some point
     selfcheck = Task("selfcheckgpt", "max-selfcheckgpt", "SelfCheckGPT")
-    mmlu = Task("mmlu", "acc", "MMLU/Acc (5-shot)")
+    mmlu = Task("mmlu", "acc", "MMLU") #MMLU/Acc (5-shot)
 
 
 # These classes are for user facing column names,
@@ -71,20 +76,22 @@ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "ma
 # # auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Avg", "number", True)])
 
 # Inference framework
-auto_eval_column_dict.append(["inference_framework", ColumnContent, ColumnContent("Inference framework", "str", True)])
+auto_eval_column_dict.append(["inference_framework", ColumnContent, ColumnContent(f"{InFrame}", "str", True)])
 
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
     # System performance metrics
-    auto_eval_column_dict.append([f"{task.name}_end_to_end_time", ColumnContent, ColumnContent(f"{task.value.col_name} End-to-end time (s)", "number", True)])
-    auto_eval_column_dict.append([f"{task.name}_prefilling_time", ColumnContent, ColumnContent(f"{task.value.col_name} Prefilling time (s)", "number", True)])
-    auto_eval_column_dict.append([f"{task.name}_decoding_throughput", ColumnContent, ColumnContent(f"{task.value.col_name} Decoding throughput (tok/s)", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_end_to_end_time", ColumnContent, ColumnContent(f"{task.value.col_name}-{E2Es}", "number", True)])
+    if task.value.benchmark in MULTIPLE_CHOICEs:
+        continue
+    auto_eval_column_dict.append([f"{task.name}_prefilling_time", ColumnContent, ColumnContent(f"{task.value.col_name}-{PREs}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_decoding_throughput", ColumnContent, ColumnContent(f"{task.value.col_name}-{TS}", "number", True)])
 
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
 auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", True)])
 auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
 auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
@@ -144,7 +151,7 @@ class InferenceFramework(Enum):
 
     def to_str(self):
         return self.value.name
-    
+
     @staticmethod
     def from_str(inference_framework: str):
         if inference_framework in ["moe-infinity"]:
@@ -152,7 +159,7 @@ class InferenceFramework(Enum):
         if inference_framework in ["hf-chat"]:
             return InferenceFramework.HF_Chat
         return InferenceFramework.Unknown
-    
+
 
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")

src/populate.py

@@ -12,7 +12,7 @@ from src.leaderboard.read_evals import get_raw_eval_results, EvalResult, update_
 
 from src.backend.envs import Tasks as BackendTasks
 from src.display.utils import Tasks
-
+from src.display.utils import E2Es, PREs, TS
 
 def get_leaderboard_df(
     results_path: str,
@@ -47,9 +47,9 @@ def get_leaderboard_df(
 
     # bm_to_name_map = {bm: name for name, bm in name_to_bm_map.items()}
     system_metrics_to_name_map = {
-        "end_to_end_time": "End-to-end time (s)",
-        "prefilling_time": "Prefilling time (s)",
-        "decoding_throughput": "Decoding throughput (tok/s)",
+        "end_to_end_time": f"{E2Es}",
+        "prefilling_time": f"{PREs}",
+        "decoding_throughput": f"{TS}",
     }
 
     all_data_json = []