added info & about descriptions, fixed model types

app.py

@@ -143,7 +143,7 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("🏅MindShift LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
                 with gr.Column():
                     with gr.Row():

src/about.py

@@ -30,20 +30,70 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">MindShift leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">MindShift: Analyzing LLMs Reactions to Psychological Prompts</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+Welcome to the leaderboard of the MindShift!
+
+Have you ever wondered how you can measure how much your LLM is following the role it has been given? Or how depressed or optimistic it is?
+
+For this purpose, we offer you a handy tool - 🏆MindShift.
+
+🏆MindShift - is a benchmark for assessing the psychological susceptibility of LLMs, such as perception, recognition and role performance with psychological characteristics. It is based on an AI model adaptation of the human psychometric person-oriented test (Minnesota Multiphasic Personality Inventory (MMPI)). 
+
+It is easy to use and can assess any LLM - both instructively tuned and in its basic version. Its scales, which are easily interpreted by humans, allow you to choose the appropriate language model for your conversational assistant or a game NPC.
+
+🤗More details on the measurement approach, roles and psychological biases can be found in the '📝About' tab. See also the paper (🚀coming soon!).
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = f"""
-## How it works
+Large language models (LLMs) hold the potential to absorb and reflect personality traits and attitudes specified by users. 
+
+<div style='display: flex; align-items: center; justify-content: center; text-align: center;'>
+        <img src='https://github.com/IrinaArmstrong/MindShift/blob/master/figs/mindshift-concept.png' style='width: 600px; height: auto; margin-right: 10px;' />
+</div>
+
+## How it works?
+
+### Questions & Scales
+To reliably validate the implicit understanding of psychological personality traits in LLMs, it is crucial to adapt psychological interpretations of the scales and formulate questions specific to the language models. When asked explicit questions about inner worlds, morality, and behavioral patterns, LLMs may exhibit biased behaviors due to extensive alignment tuning. This can result in inconsistent and unrepresentative questionnaire outcomes.
+
+To assess the susceptibility of LLMs to personalization, we utilized the Standardized Multifactorial Method for Personality Research (SMMPR), which is based on the Minnesota Multiphasic Personality Inventory (MMPI). It is a questionnaire-based test consisting of 566 short statements that individuals rate as true or false for themselves. 
+The test assesses psychological characteristics on 10 basic "personality profile" scales, named after the nosological forms of corresponding disorders: 
+* Hypochondria (Hs), 
+* Depression (D), 
+* Emotional Lability (Hy), 
+* Psychopathy (Pd), 
+* Masculinity-Femininity (Mf), 
+* Rigidity/Paranoia (Pa), 
+* Anxiety/Psychasthenia (Pf), 
+* Individualism/Schizophrenia (Sc), 
+* Optimism (Ma), 
+* Social Introversion (Si). 
+
+Additionally, the test includes three validation scales to assess the truthfulness and sincerity of the respondent's answers: Lie (L), Infrequency (F), and Defensiveness (D).
+
+To ensure the reproducibility of our methodology for both instructively tuned and basic versions, we leveraged the LLM's ability to complete textual queries. We constructed a set of statements from the questionnaire and asked LLM to finish the prompt with only one option: True or False.
+
+<div style='display: flex; align-items: center; justify-content: center; text-align: center;'>
+        <img src='https://github.com/IrinaArmstrong/MindShift/blob/master/figs/mindshift-statements.png' style='width: 600px; height: auto; margin-right: 10px;' />
+</div>
+
+### Psychological prompts
+
+To measure the extent to which an LLM understands personality, MindShift at its core contains a structured method for introducing psychologically oriented biases into prompts. 
+Introducing specific personality traits into an LLM can be achieved by providing it with a natural language description of the persona. In our methodology, the persona description consists of two parts: the Persona General Descriptor and the Psychological Bias Descriptor. The Persona General Descriptor includes general statements about the character's lifestyle, routines, and social aspects, while the Psychological Bias Descriptor covers specific psychological attitudes with varying degrees of intensity.
+
+<div style='display: flex; align-items: center; justify-content: center; text-align: center;'>
+        <img src='https://github.com/IrinaArmstrong/MindShift/blob/master/figs/mindshift-input-schema.png' style='width: 600px; height: auto; margin-right: 10px;' />
+</div>
 
-## Reproducibility
-To reproduce our results, here is the commands you can run:
+They are combined with Persona General Descriptor - a full character role (including gender, age, marital status, personal circumstances, hobbies, etc.), sampled from PersonaChat dialogue dataset. Together they form a complete description of the persona.  
 
+### Paper
+You can find more details about the assessment, a list of psychological prompts, roles and experiments in the paper (🚀coming soon!).
 """
 
 EVALUATION_QUEUE_TEXT = """
@@ -78,4 +128,5 @@ If everything is done, check you can launch the EleutherAIHarness on your model
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""
+(🚀coming soon!)
 """

src/leaderboard/read_evals.py

@@ -47,6 +47,12 @@ class EvalResult:
         org_and_model = config.get("model_name", config.get("model_args", None))
         org_and_model = org_and_model.split("/", 1)
 
+        model_type = ModelType.Unknown
+        if ("instruct" in org_and_model[-1].lower()) or ("-it" in org_and_model[-1].lower()):
+            model_type = ModelType.from_str("instruction-tuned")
+        else:
+            model_type = ModelType.from_str("pretrained")
+
         if len(org_and_model) == 1:
             org = None
             model = org_and_model[0]
@@ -85,7 +91,8 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
-            precision=precision,  
+            precision=precision,
+            model_type=model_type,
             revision=config.get("model_sha", ""),
             still_on_hub=still_on_hub,
             architecture=architecture

src/submission/submit.py

@@ -1,119 +1,119 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )
+# import json
+# import os
+# from datetime import datetime, timezone
+#
+# from src.display.formatting import styled_error, styled_message, styled_warning
+# from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+# from src.submission.check_validity import (
+#     already_submitted_models,
+#     check_model_card,
+#     get_model_size,
+#     is_model_on_hub,
+# )
+#
+# REQUESTED_MODELS = None
+# USERS_TO_SUBMISSION_DATES = None
+#
+# def add_new_eval(
+#     model: str,
+#     base_model: str,
+#     revision: str,
+#     precision: str,
+#     weight_type: str,
+#     model_type: str,
+# ):
+#     global REQUESTED_MODELS
+#     global USERS_TO_SUBMISSION_DATES
+#     if not REQUESTED_MODELS:
+#         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+#
+#     user_name = ""
+#     model_path = model
+#     if "/" in model:
+#         user_name = model.split("/")[0]
+#         model_path = model.split("/")[1]
+#
+#     precision = precision.split(" ")[0]
+#     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+#
+#     if model_type is None or model_type == "":
+#         return styled_error("Please select a model type.")
+#
+#     # Does the model actually exist?
+#     if revision == "":
+#         revision = "main"
+#
+#     # Is the model on the hub?
+#     if weight_type in ["Delta", "Adapter"]:
+#         base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+#         if not base_model_on_hub:
+#             return styled_error(f'Base model "{base_model}" {error}')
+#
+#     if not weight_type == "Adapter":
+#         model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+#         if not model_on_hub:
+#             return styled_error(f'Model "{model}" {error}')
+#
+#     # Is the model info correctly filled?
+#     try:
+#         model_info = API.model_info(repo_id=model, revision=revision)
+#     except Exception:
+#         return styled_error("Could not get your model information. Please fill it up properly.")
+#
+#     model_size = get_model_size(model_info=model_info, precision=precision)
+#
+#     # Were the model card and license filled?
+#     try:
+#         license = model_info.cardData["license"]
+#     except Exception:
+#         return styled_error("Please select a license for your model")
+#
+#     modelcard_OK, error_msg = check_model_card(model)
+#     if not modelcard_OK:
+#         return styled_error(error_msg)
+#
+#     # Seems good, creating the eval
+#     print("Adding new eval")
+#
+#     eval_entry = {
+#         "model": model,
+#         "base_model": base_model,
+#         "revision": revision,
+#         "precision": precision,
+#         "weight_type": weight_type,
+#         "status": "PENDING",
+#         "submitted_time": current_time,
+#         "model_type": model_type,
+#         "likes": model_info.likes,
+#         "params": model_size,
+#         "license": license,
+#         "private": False,
+#     }
+#
+#     # Check for duplicate submission
+#     if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+#         return styled_warning("This model has been already submitted.")
+#
+#     print("Creating eval file")
+#     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+#     os.makedirs(OUT_DIR, exist_ok=True)
+#     out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+#
+#     with open(out_path, "w") as f:
+#         f.write(json.dumps(eval_entry))
+#
+#     print("Uploading eval file")
+#     API.upload_file(
+#         path_or_fileobj=out_path,
+#         path_in_repo=out_path.split("eval-queue/")[1],
+#         repo_id=QUEUE_REPO,
+#         repo_type="dataset",
+#         commit_message=f"Add {model} to eval queue",
+#     )
+#
+#     # Remove the local file
+#     os.remove(out_path)
+#
+#     return styled_message(
+#         "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+#     )