update readme

adhoc.py

@@ -1,6 +1,6 @@
 import asyncio
 from src.backend.manage_requests import EvalRequest
-from src.envs import LIMIT, EVAL_RESULTS_PATH_BACKEND, RESULTS_REPO, DEVICE
+from src.envs import LIMIT, EVAL_RESULTS_PATH_BACKEND, RESULTS_REPO, DEVICE, LOCAL_MODEL_NAME
 from src.backend.run_eval_suite import run_evaluation
 from src.about import HarnessTasks
 
@@ -25,8 +25,7 @@ async def run_adhoc_eval(eval_request: EvalRequest):
 
 def main():
     # eval_request: EvalRequest(model='meta-llama/Llama-2-7b-chat-hf', private=False, status='FINISHED', json_filepath='', weight_type='Original', model_type='\ud83d\udfe2 : pretrained', precision='bfloat16', base_model='', revision='main', submitted_time='2023-11-21T18:10:08Z', likes=0, params=0.1, license='custom')
-    model_name = "2cce1211d4e3aa176d5688ab38b85f46b41be221f856f4171e6e42b55864f3f3"
-    vals = {"model": model_name, "json_filepath": "", "base_model": "", "revision": "main",
+    vals = {"model": LOCAL_MODEL_NAME, "json_filepath": "", "base_model": "", "revision": "main",
             "private": False,
             "precision": "bfloat16", "weight_type": "Original", "status": "PENDING",
             "submitted_time": "2023-11-21T18:10:08Z", "model_type": "\ud83d\udfe2 : pretrained", "likes": 0,

run-adhoc.sh

@@ -11,5 +11,17 @@ set -Eeuoxa pipefail
 
 LOCAL_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
+for ARGUMENT in "$@"
+do
+   KEY=$(echo $ARGUMENT | cut -f1 -d=)
+
+   KEY_LENGTH=${#KEY}
+   VALUE="${ARGUMENT:$KEY_LENGTH+1}"
+
+   export "$KEY"="$VALUE"
+done
+
+echo "Model passed is: $LOCAL_MODEL_NAME"
+
 docker buildx build --platform=linux/amd64 -t adhoc .
-docker run -it --rm --platform=linux/amd64 -v $LOCAL_DIRECTORY/output-data:/code/data adhoc python adhoc.py
+docker run -it --rm --platform=linux/amd64 -v $LOCAL_DIRECTORY/output-data:/code/data -e LOCAL_MODEL_NAME=$LOCAL_MODEL_NAME adhoc python adhoc.py

src/about.py

@@ -59,27 +59,40 @@ This leaderboard evaluates and benchmarks LLM hallucinations when answering ques
 
 # Which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = f"""
+## About the leaderboard
+
+This leaderboard ranks different models (both pre-trained and finetuned) on their **Photographic Memory** as measured on the **Lamini Evaluation Suite**. 
+**Photographic Memory** means the ability of a model to recall and generate accurate information learnt during pre-training or finetuning.
+The benchmarks are divided into two categories: standard benchmarks and custom benchmarks. The standard benchmarks evaluate the model's generation abilities on standard data, while the custom benchmarks evaluate the model's performance on domain-specific data.
+
 ## How it works
 
-All the requests are executed on the Lamini servers. Get your API key from [Lamini](https://app.lamini.ai/) today!
+All the open-source models evaluation requests are executed on the Lamini servers. Get your API key from [Lamini](https://app.lamini.ai/) today!
+
+### Standard benchmarks:
 
-We run the following evaluation metrics:
+We run the following standard evaluation benchmarks to assess any regression from the LLM pretraining phase:
 
-1. **MMLU (Global facts)**: Evaluates the model's ability to answer questions about global facts.
-2. **MMLU (Formal logic)**: Evaluates the model's ability to answer questions about formal logic.
-3. **Truthful QA**: Evaluates the model's ability to generate truthful answers.
+**1. MMLU (Global facts)**: Evaluates the model's ability to answer questions about global facts.
+
+**2. Truthful QA**: Evaluates the model's ability to generate truthful answers.
 
 ### Custom benchmarks:
 
-We introduce a few custom benchmarks run on an open-source product catalog dataset.
+We also introduce a few custom benchmarks for domain specific data to assess precision and recall for numbers and exact figures. The same can be adapted to other domains.
+
+### <ins>Domain 1: E-commerce</ins>
+Given a product catalog dataset, we evaluate the model on its ability to remember the product information correctly and answer a customer's question.
+The expectation here is that the models finetuned over this dataset should be able to generate the correct product information compared to the base model. We assign the following scores to the model output:
+
+**3. Product Response Subjective Score**: Subjectively score and evaluates the model's ability to answer a customer's question. The answer is judged on proximity to truth and relevance.
 
-4. **Subjective Response Score**: Subjectively score and evaluates the model's ability to answer a customer's question about a product.
-5. **Product ID Precision Score**: An exact match of the product id answered by the model in response to a customer's question. This evaluates how well the model learns and its recall abilities.
+**4. Product ID Precision Score**: An exact match of the product id answered by the model in response to a customer enquiring about a product. This evaluates how well the model learns and can recall the exact figures learnt during training.
 
+The evaluation dataset for e-commerce domain is available [here](https://github.com/lamini-ai/leaderboard/blob/main/src/datasets/shopping.jsonl).
+
+The data for finetuning models was created using the [Lamini generation pipeline](https://github.com/lamini-ai/instacart-greg/blob/main/instacart/generate_data_pipeline.py) and is available [here](https://huggingface.co/datasets/lamini/product-catalog-questions).
 
-Task:
-Given a product catalog dataset, we evaluate the models on its ability to remember the product information correctly and answer a customer's question based on the product information.
-The expectation here is that the models finetuned over this dataset should be able to generate the correct product information compared to the base model.
 
 Prompt used to generate product information:
 ```
@@ -91,10 +104,40 @@ The customer asks\n
 <question>
 ```
 
+### <ins>Domain 2: Medicine</ins>
+
+Here we mimic a common problem in the medical domain - accurately answering questions about the ICD-11 standard. ICD-11 (International Classification of Diseases 11th Revision) is a globally used standard for classifying diseases, health conditions, and related phenomena. 
+It is maintained by the World Health Organization (WHO) and serves as a common language for health information systems, epidemiology, health statistics, clinical care, and research. 
+
+**5. ICD Code Precision Score**: When a model answers a question about an ICD11 standard, this metric evaluates the accuracy of the ICD11 code generated. The standard is pretty stringent so it is important to get the right code.
+
+**6. ICD Code Subjective Score**: This metric evaluates the overall quality of the answers in terms of correctness and completeness, and assigns a score to it.
+
+
+The evaluation dataset for medical domain is available [here](https://github.com/lamini-ai/leaderboard/blob/main/src/datasets/icd11.jsonl).
+
+The data for finetuning models was created using the [Lamini generation pipeline](https://github.com/lamini-ai/lamini-sdk/blob/greg.cpt-gpt/04_IFT/generate_data_pipeline.py) and is available [here](https://huggingface.co/datasets/lamini/icd-11-qa).
+
+
+### <ins>Domain 3: Finance</ins>
+
+Here we simulate a problem in the finance field - accurately answering questions about companies' financial performance based on the transcripts of their earnings calls. 
+
+**7. Earnings Value Precision Score**: While answering the question about companies financials, the model might output some values like 800M $. This score evaluates the accuracy of the value and the units generated based on the transcripts of the calls aka the training data.
+
+**8. Earnings Value Subjective Score:**: This metric assigns a score to the overall quality of the answers in terms of correctness and coherence.
+
+
+The evaluation dataset for finance domain is available [here](https://github.com/lamini-ai/leaderboard/blob/main/src/datasets/earnings_calls.jsonl).
+
+The data for finetuning models was created using the [Lamini generation pipeline](https://github.com/lamini-ai/lamini-earnings-calls) and is available [here](https://huggingface.co/datasets/lamini/earnings-calls-qa).
+
+
+### Scoring
 
 We use Mistral(mistralai/Mistral-7B-Instruct-v0.2) to score the model answers.
 
-Prompt to get the scoring rubric:
+Sample scoring rubric for the e-commerce domain. Rest of the domain rubrics are similar to this:
 ```
 Read this scoring rubric carefully and follow the instructions precisely:\n
 A score of 5 means that model's id is the same as the gold answer's id.\n
@@ -110,7 +153,7 @@ gold answer == Product ID: 5678, Product Name: Tuna, Description: Canned Tuna, m
 Assign a 5 even if fields are missing, for example: gold answer == Product ID: 1234, model response == Product ID: 1234, score == 5\n
 ```
 
-Prompt used to score the model answers:
+Sample prompt used to score the model answers. Rest of the domain prompts are similar to this:
 ```
 A model is going to answer a question. Your job is to score the answer, comparing it to a golden reference. You are an expert scorer.\n<</SYS>>\n\n
 Rate the answer using a score from 1 (lowest match) to 5 (highest match).\n
@@ -132,25 +175,28 @@ f"How would you score the model's answer compared to the gold answer (using the
 ## Reproducibility
 1. Clone this repo by following the steps provided [here.](https://huggingface.co/spaces/lamini/leaderboard?clone=true)
 
-2. Now change the environment variables in `src/env`.
+2. Now change the environment variables in `src/env` to add your api keys.
 
 There are 2 ways to run this code locally:
 
 1. To run the backend server + UI locally, run the following command:
 ```
-./start.sh RUN_MODE="LOCAL" LOCAL_MODEL_NAME="<any huggingface model you want to try>"
+./start.sh RUN_MODE="LOCAL" LOCAL_MODEL_NAME="<any huggingface or openai model you want to try>"
 
 Eg:
 ./start.sh RUN_MODE="LOCAL" LOCAL_MODEL_NAME="hf-internal-testing/tiny-random-gpt2"
 ```
 
-2. To run only the backend server locally, make changes to adhoc.py to specify your model name and run the following command:
+2. To run only the backend server locally to quickly get evaluation results, run the following command:
 ```
-./run-adhoc.sh
+./run-adhoc.sh LOCAL_MODEL_NAME="<any huggingface or openai model you want to try>"
 ```
 
+This will still allow you to save your benchmarking results in the folder `output-data`.
+
 """
 
+# We don't allow model submission right now.
 EVALUATION_QUEUE_TEXT = """
 ## Some good practices before submitting a model
 
@@ -185,13 +231,13 @@ CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""
 @dataset{Lamini,
   author       = {Ayushi Sharma},
-  title        = {Lamini Anti-hallucination Leaderboard},
+  title        = {Lamini Photographic Memory Leaderboard},
   year         = {2024},
   month        = {03},
   publisher    = {Lamini},
   doi          = {},
   url          = {},
-  abstract     = {This leaderboard evaluates and benchmarks LLM hallucinations when answering questions. Models may be pre-trained or finetuned.},
-  keywords     = {llm, hallucination},
-  license      = {Apache-2.0},
+  abstract     = {},
+  keywords     = {llm, hallucination, photographic memory},
+  license      = {cc-by-4.0},
 }"""

src/backend/custom_evaluator.py

@@ -45,7 +45,7 @@ class CustomEvaluator:
             dict: A dictionary containing evaluation results.
         """
         try:
-            if "gpt" in self.model:
+            if "gpt4" in self.model:
                 model_type = "gpt4"
             else:
                 model_type = "lamini"

src/backend/run_eval_suite.py

@@ -19,6 +19,7 @@ async def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, bat
             "WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT."
         )
 
+    print(f"\nRunning eval_request: {eval_request}\n")
     custom_evaluator = CustomEvaluator(eval_request.model, eval_request.revision, eval_request.precision,
                                        batch_size, device, no_cache, limit, write_out=True,
                                        output_base_path='logs')

src/envs.py

@@ -6,16 +6,16 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("TOKEN")
 RUN_MODE = os.environ.get("RUN_MODE")
-LOCAL_MODEL_NAME = os.environ.get("LOCAL_MODEL_NAME")
+LOCAL_MODEL_NAME = os.environ.get("LOCAL_MODEL_NAME", "hf-internal-testing/tiny-random-gpt2")
 
 LAMINI_ENV = os.environ.get("LAMINI_ENV", "PRODUCTION")
 LAMINI_API_KEY = os.environ.get("LAMINI_API_KEY")
 OPENAI_API_KEY =  os.environ.get("OPENAI_API_KEY")
-MAX_EXAMPLES = os.environ.get("MAX_EXAMPLES", 10)
+MAX_EXAMPLES = os.environ.get("MAX_EXAMPLES", 2)
+LIMIT = os.environ.get("MAX_EXAMPLES", 2)
 
 OWNER = "lamini" # Change to your org - don't forget to create a results and request file
 DEVICE = "cpu" # "cuda:0" if you add compute
-LIMIT = os.environ.get("MAX_EXAMPLES")
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/leaderboard"