Upload 10 files

app.py

@@ -11,6 +11,7 @@ from src.utils import (
 from src.demo import (
     generate,
     random_examples,
+    return_ground_truth,
 )
 
 
@@ -66,14 +67,21 @@ demo = gr.Blocks(css=custom_css)
 with demo:
     with gr.Row():
         gr.Markdown(
-            """<div style="text-align: center;"><h1> 🤖ConvRe🤯 <span style='color: #e6b800;'>Leaderboard</span></h1></div>\
-            <br>\
-            <p> 🤖ConvRe🤯 is the benchmark proposed in our EMNLP 2023 paper: <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"> An Investigation of LLMs’ Inefficacy in Understanding Converse Relations</a>. It aims to evaluate LLMs' ability on understanding converse relations. Converse relation is defined as the opposite of semantic relation while keeping the surface form of the triple unchanged. For example, the triple (x, has part, y) is interpreted as "x has a part called y" in normal relation, while "y has a part called x" in converse relation 🔁.
+            """<div align= "center">
+    <h1>🤖 ConvRe 🤯 <span style='color: #e6b800;'>  Leaderboard</span></h1>
+</div>
 
-            The experiments in our paper suggested that LLMs often resort to shortcut learning (or superficial correlations) and still face challenges on our 🤖ConvRe🤯 benchmark even for powerful models like GPT-4.
-            </p>""",
+""",
             elem_classes="markdown-text",
         )
+    
+    gr.Markdown("""🤖**ConvRe**🤯 is the benchmark proposed in our EMNLP 2023 main conference paper: [An Investigation of LLMs’ Inefficacy in Understanding Converse Relations]().
+It aims to evaluate LLMs' ability on understanding converse relations.
+Converse relation is defined as the opposite of semantic relation while keeping the surface form of the triple unchanged.
+For example, the triple `(x, has part, y)` is interpreted as "x has a part called y" in normal relation, while "y has a part called x" in converse relation 🔁.
+                
+The experiments in our paper suggested that LLMs often resort to shortcut learning (or superficial correlations) and still face challenges on our 🤖ConvRe🤯 benchmark even for powerful models like GPT-4.
+                """, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🔢 Data", id=0):
@@ -145,19 +153,48 @@ with demo:
                     )
 
         with gr.TabItem("Submit results 🚀", id=3):
-                    gr.Markdown("Submit Here")
+                    gr.Markdown("""<div align= "center">
+    <h1>Comming Soon ❤️</span></h1>
+</div>
+
+""")
 
     with gr.Column():
         gr.Markdown(
-            """<div style="text-align: center;"><h2> 🤖ConvRe🤯 Demo </h2></div>\
+            """<div style="text-align: center;"><h1> 🤖ConvRe🤯 Demo (Llama-2-Chat-7B🦙) </h1></div>\
             <br>\
             """,
             elem_classes="markdown-text",
         )
 
-        output_box = gr.Textbox(lines=10, max_lines=10, label="ChatBot")
+        output_box = gr.Textbox(lines=10, max_lines=10, label="Llama-2-Chat-7B Answer", interactive=False)
+
+        input_box = gr.Textbox(lines=12, max_lines=12, label="User Input")
+
+        ground_truth_display = gr.Textbox("", lines=1, max_lines=1, label="😊Correct Answer😊", interactive=False)
+        
+        with gr.Column():
+            
+            
+            with gr.Accordion("Additional Inputs", open=False):
+                sys_prompt = gr.Textbox(label="System prompt", value=DEFAULT_SYSTEM_PROMPT, lines=6)
+            
+                max_new_tokens=gr.Slider(
+                    label="Max new tokens",
+                    minimum=1,
+                    maximum=MAX_MAX_NEW_TOKENS,
+                    step=1,
+                    value=DEFAULT_MAX_NEW_TOKENS,
+                )
+
+                temperature = gr.Slider(
+                    label="Temperature",
+                    minimum=0.1,
+                    maximum=4.0,
+                    step=0.1,
+                    value=0.1,
+                )
 
-        input_box = gr.Textbox(lines=12, max_lines=12, label="Input")
 
         with gr.Row():
             re2text_easy_btn = gr.Button("Random Re2Text Easy Example 😄")
@@ -188,28 +225,13 @@ with demo:
                 outputs = input_box,
             )
 
-        with gr.Accordion("Additional Inputs", open=False):
-            sys_prompt = gr.Textbox(label="System prompt", value=DEFAULT_SYSTEM_PROMPT, lines=6)
-            
-            gr.Slider(
-                label="Max new tokens",
-                minimum=1,
-                maximum=MAX_MAX_NEW_TOKENS,
-                step=1,
-                value=DEFAULT_MAX_NEW_TOKENS,
-            )
-
-            gr.Slider(
-                label="Temperature",
-                minimum=0,
-                maximum=4.0,
-                step=0.05,
-                value=0,
-            )
-
         with gr.Row():
             gr.ClearButton([input_box, output_box])
-            submit_btn = gr.Button("Submit")
-            submit_btn.click(generate, inputs=[input_box, sys_prompt], outputs=[output_box])
+            submit_btn = gr.Button("Submit🏃")
+            submit_btn.click(generate, inputs=[input_box, sys_prompt, temperature, max_new_tokens], outputs=[output_box])
+
+            answer_btn = gr.Button("Answer🤔")
+            answer_btn.click(return_ground_truth, inputs=[], outputs=[ground_truth_display])
+
 
-demo.launch()
+demo.queue(max_size=32).launch(enable_queue=True)

data/eval_board.csv

@@ -1,7 +1,11 @@
-Models,Re2Text-Easy,Text2Re-Easy,Re2Text-Hard,Text2Re-Hard,Avg,Links
-gpt-4-0314,99.0,94.1,15.6,17.7,56.6,https://openai.com/research/gpt-4
-gpt-3.5-turbo,83.4,60.8,16.0,39.4,49.9,https://chat.openai.com/
-text-davinci-003,85.7,84.0,23.7,34.4,57.0,https://platform.openai.com/docs/models/gpt-3-5
-llama-2-7b-chat-hf,0,0,0,0,0,https://huggingface.co/meta-llama/Llama-2-7b-chat-hf
-qwen-7b-chat,0,0,0,0,0,https://huggingface.co/Qwen/Qwen-7B-Chat
-internlm-7b-chat,0,0,0,0,0,https://huggingface.co/internlm/internlm-chat-7b
+Models,Re2Text-Easy,Text2Re-Easy,Re2Text-Hard,Text2Re-Hard,Avg,Model Size,Links
+gpt-4-0314,98.7,93.6,16.4,17.1,56.5,unknown,https://openai.com/research/gpt-4
+gpt-3.5-turbo-0301,83.5,60.7,59.0,39.0,60.6,unknown,https://chat.openai.com/
+text-davinci-003,85.4,83.8,55.8,34.8,65.0,175B,https://platform.openai.com/docs/models/gpt-3-5
+claude-instant-1.1,65.7,87.2,52.3,26.2,57.9,unknown,https://www.anthropic.com/index/introducing-claude
+claude-1.3,89.7,82.3,37.3,56.6,66.5,unknown,https://www.anthropic.com/index/introducing-claude
+flan-t5-xxl,79.4,96.8,20.7,4.8,50.4,11B,https://huggingface.co/google/flan-t5-xxl
+flan-t5-xl,91.5,90.6,7.9,17.8,52.0,3B,https://huggingface.co/google/flan-t5-xl
+flan-t5-large,71.5,77.3,26.2,29.6,51.2,780M,https://huggingface.co/google/flan-t5-large
+flan-t5-base,84.6,51.2,17.0,50.2,50.8,250M,https://huggingface.co/google/flan-t5-base
+flan-t5-small,51.8,50.1,46.5,49.5,49.5,60M,https://huggingface.co/google/flan-t5-small

src/css_html.py

@@ -12,6 +12,10 @@ custom_css = """
     font-size: 16px !important;
 }
 
+#answer-text {
+    font-size: 28px !important;
+}
+
 #models-to-add-text {
     font-size: 18px !important;
 }

src/demo.py

@@ -6,12 +6,12 @@ from typing import Iterable
 import torch
 from huggingface_hub import HfApi
 from datasets import load_dataset
-from transformers import T5Tokenizer, T5ForConditionalGeneration
-from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
 
 
-TOKEN = os.environ.get("HF_TOKEN", None)
+ground_truth = ""
 
+TOKEN = os.environ.get("HF_TOKEN", None)
 
 type2dataset = {
     "re2text-easy": load_dataset('3B-Group/ConvRe', "en-re2text", token=TOKEN, split="prompt1"),
@@ -24,10 +24,15 @@ model_id = "meta-llama/Llama-2-7b-chat-hf"
 tokenizer = AutoTokenizer.from_pretrained(model_id, token=TOKEN)
 model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, token=TOKEN, device_map="auto").eval()
 
+
+# model_id = "google/flan-t5-base"
+# tokenizer = T5Tokenizer.from_pretrained(model_id)
+# model = T5ForConditionalGeneration.from_pretrained(model_id, device_map="auto")
+
 # type2dataset = {}
 
 
-def generate(input_text, sys_prompt) -> str:
+def generate(input_text, sys_prompt, temperature, max_new_tokens) -> str:
     sys_prompt = f'''[INST] <<SYS>>
 {sys_prompt}
 <</SYS>>
@@ -35,24 +40,44 @@ def generate(input_text, sys_prompt) -> str:
 '''
     input_str = sys_prompt + input_text + " [/INST]"
     
-    input_ids = tokenizer(input_str, return_tensors="pt").input_ids.to('cuda')
-    outputs = model.generate(input_ids, max_length=512)
+    input_ids = tokenizer(input_str, return_tensors="pt").to('cuda')
 
-    result = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
+    streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
 
-    result = result.split(' [/INST]')
-    result = result[0] + '\n\n' + result[1]
+    generate_kwargs = dict(
+        input_ids,
+        streamer=streamer,
+        max_new_tokens=max_new_tokens,
+        do_sample=True,
+        temperature=float(temperature)
+    )
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
 
-    return result
+    # Pull the generated text from the streamer, and update the model output.
+    model_output = ""
+    for new_text in streamer:
+        model_output += new_text
+        yield model_output
+    return model_output
 
 
 def random_examples(dataset_key) -> str:
+    
+
     # target_dataset = type2dataset[f"{task.lower()}-{type.lower()}"]
     target_dataset = type2dataset[dataset_key]
 
     idx = random.randint(0, len(target_dataset) - 1)
     item = target_dataset[idx]
+
+    global ground_truth
+    ground_truth = item['answer']
+
     return item['query']
 
 
+def return_ground_truth() -> str:
+    correct_answer = ground_truth
+    return correct_answer
 

src/utils.py

@@ -22,6 +22,7 @@ class AutoEvalColumn:  # Auto evals column
     re2text_hard = ColumnContent("Re2Text-Hard", "number", True)
     text2re_hard = ColumnContent("Text2Re-Hard", "number", True)
     avg = ColumnContent("Avg", "number", True)
+    model_size = ColumnContent("Model Size", "markdown", True)
 
     link = ColumnContent("Links", "str", False)