update

app.py

@@ -0,0 +1,107 @@
+import gradio as gr
+import sys
+import os
+from datasets import load_dataset
+from typing import List
+import utils
+
+
+DESCRIPTIONS = """# 🐯TIGERScore
+
+We present ***TIGERScore***, a **T**rained metric that follows **I**nstruction **G**uidance to perform **E**xplainable, and **R**eference-free evaluation over a wide spectrum of text generation tasks. Different from other automatic evaluation methods that only provide arcane scores, TIGERScore is guided by the natural language instruction to provide error analysis to pinpoint the mistakes in the generated text.
+
+### [**Website**](https://tiger-ai-lab.github.io/TIGERScore/)  [**Paper**](https://arxiv.org/abs/2310.00752)   [**Code**](https://github.com/TIGER-AI-Lab/TIGERScore)   [**TIGERScore-7B**](https://huggingface.co/TIGER-Lab/TIGERScore-7B-V1.0)   [**TIGERScore-13B**](https://huggingface.co/TIGER-Lab/TIGERScore-13B-V1.0)
+
+"""
+
+EXAMPLES_DATASET = load_dataset("TIGER-Lab/MetricInstruct", split="train_mix")
+SHUFFLED_EXAMPLES_DATASET = EXAMPLES_DATASET.shuffle(seed=42)
+EXAMPLES = []
+fields = ["task", "instruction", "input_context", "hypo_output"]
+print("Loading examples...")
+for i, ex in enumerate(SHUFFLED_EXAMPLES_DATASET):
+    if any([not ex[field] for field in fields]):
+        continue
+    EXAMPLES.append([ex[field] for field in fields])
+    if i >= 100:
+        break
+
+def tigerscore(task, input_context, generation_instruction, hypo_output, max_new_tokens=512, temperature=0.7, top_p=1.0):
+    return utils.generate(
+        task, input_context, 
+        generation_instruction, hypo_output, 
+        max_new_tokens=max_new_tokens, 
+        temperature=temperature, top_p=top_p
+    )
+
+def get_examples(task, inst_textbox, input_textbox, hypo_output_textbox):
+    return gr.Dropdown.update(value=task), inst_textbox, input_textbox, hypo_output_textbox
+
+## initialize the model
+print("Loading TIGERScore model...")
+utils.load_tigerscore("7b")
+
+with gr.Blocks(theme='gradio/soft') as demo:
+    gr.Markdown(DESCRIPTIONS)
+    gr.Markdown("## TIGERScore Inputs")
+
+    tasks_dropdown = gr.Dropdown(label="Task", choices=utils.tasks + ["other"], value="translation", show_label=True)
+    inst_textbox = gr.Textbox(lines=1, label="Instruction", placeholder="Enter instruction here", show_label=True)
+    input_textbox = gr.Textbox(lines=4, label="Input Context", placeholder="Enter input context here", show_label=True)
+    hypo_output_textbox = gr.Textbox(lines=4, label="Hypothesis Output", placeholder="Enter hypothesis output to be evaluated here", show_label=True)
+    
+    with gr.Row():
+        clear_button = gr.Button('Clear', variant='primary')
+        submit_button = gr.Button('Submit', variant='primary')
+    
+    with gr.Accordion(label='Advanced options', open=False):
+        max_new_tokens = gr.Slider(
+            label='Max new tokens fuser can generate',
+            minimum=256,
+            maximum=1024,
+            step=1,
+            value=512,
+        )
+        temperature = gr.Slider(
+            label='Temperature of fuser generation',
+            minimum=0.1,
+            maximum=2.0,
+            step=0.1,
+            value=0.7,
+        )
+        top_p = gr.Slider(
+            label='Top-p of fuser generation',
+            minimum=0.05,
+            maximum=1.0,
+            step=0.05,
+            value=1.0,
+        )
+
+    gr.Markdown("## TIGERScore Outputs")
+    evaluation_output_textbox = gr.Textbox(lines=4, label="Evaluation Output", placeholder="Evaluation output", show_label=True)
+
+
+    submit_button.click(
+        fn=tigerscore,
+        inputs=[tasks_dropdown, input_textbox, inst_textbox, hypo_output_textbox, max_new_tokens, temperature, top_p],
+        outputs=evaluation_output_textbox,
+    )
+    
+    batch_examples = gr.Examples(
+        examples=EXAMPLES,
+        fn=get_examples,
+        cache_examples=True,
+        examples_per_page=5,
+        inputs=[tasks_dropdown, inst_textbox, input_textbox, hypo_output_textbox],
+        outputs=[tasks_dropdown, inst_textbox, input_textbox, hypo_output_textbox],
+    )  
+
+    citations = gr.Markdown("""## Citation
+@article{jiang2023TIGERScore,
+  title={TIGERScore: Towards Building Explainable Metric for All Text Generation Tasks},
+  author={Dongfu Jiang, Yishan Li, Ge Zhang, Wenhao Huang, Bill Yuchen Lin, Wenhu Chen},
+  journal={arXiv preprint arXiv:2310.00752},
+  year={2023}
+}""")
+
+demo.queue(max_size=20).launch()

requirements.txt

@@ -0,0 +1,5 @@
+transformers==4.33.2
+datasets==2.10.0
+torch
+accelerate
+gradio

utils.py

@@ -0,0 +1,85 @@
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from string import Template
+import torch
+
+FINETUNE_INST = "You are evaluating errors in a model-generated output for a(an) ${task} task."
+FINETUNE_INPUT = """\
+Task instruction: ${generation_instruction}
+Source: ${input_context}
+Model-generated Output: ${hypothesis_output}
+
+Based on the given task instruction and source, identify errors in this model-generated output.
+For each error you give in the response, please also elaborate the following information:
+- error location (the words that are wrong in the output)
+- error aspect it belongs to.
+- explanation why it's an error, and the correction suggestions.
+- severity of the error ("Major" or "Minor"). 
+- reduction of score (between 0.5 and 5 given the severity of the error)
+
+Your evaluation output:
+"""
+
+TIGERScore_model_map = {
+    "7b": "TIGER-Lab/TIGERScore-7B-V1.0",
+    "13b": "TIGER-Lab/TIGERScore-13B-V1.0",
+}
+tigerscore_model = None
+tigerscore_tokenizer = None
+
+tasks = [
+    "translation",
+    "summarization",
+    "data2text",
+    "mathQA",
+    "long-form QA",
+    "instruction-following",
+]
+
+def load_tigerscore(model_size):
+    assert model_size in TIGERScore_model_map
+    model_name = TIGERScore_model_map[model_size]
+    global tigerscore_model, tigerscore_tokenizer
+    tigerscore_model = AutoModelForCausalLM.from_pretrained(
+        model_name,
+        torch_dtype=torch.bfloat16,
+        device_map="auto"
+    )
+    tigerscore_tokenizer = AutoTokenizer.from_pretrained(
+        model_name,
+        use_fast=True
+    )
+
+def generate(task, input_context, generation_instruction, hypo_output, **generate_kwargs):
+    inst_part = Template(FINETUNE_INST)
+    inst_part = inst_part.substitute(task=task)
+    input_part = Template(FINETUNE_INPUT)
+    input_part = input_part.substitute(
+        generation_instruction=generation_instruction,
+        input_context=input_context,
+        hypothesis_output=hypo_output
+    )
+    prompt = (inst_part + "\n" + input_part).strip("\n ") + "\n"
+    encodings = tigerscore_tokenizer(prompt, return_tensors="pt")
+    input_ids = encodings["input_ids"].to(tigerscore_model.device)
+    attention_mask = encodings["attention_mask"].to(tigerscore_model.device)
+    gen_params = {
+        "input_ids": input_ids,
+        "attention_mask": attention_mask,
+        "max_new_tokens": 512,
+        "do_sample": True,
+        "top_k": 1,
+        "num_return_sequences": 1,
+    }
+    gen_params.update(generate_kwargs)
+    output = tigerscore_model.generate(**gen_params)
+    output = tigerscore_tokenizer.decode(output[0][len(input_ids[0]):], skip_special_tokens=True)
+    return output
+
+if __name__ == "__main__":
+    task = "translation"
+    input_context = "Der künftige EM-Cheforganisator Philipp Lahm soll laut Grindel im DFB-Präsidium mitarbeiten."
+    generation_instruction = "Translate the following text from German to English."
+    hypo_output = "According to Grindel, the future head of the European Championships, Philipp Lahm, is to participate in the DFB Presidency."
+    output = generate(task, input_context, generation_instruction, hypo_output)
+    print(output)
+