Spaces:
Running
Running
| import gradio as gr | |
| from executors import benchmark_executor, synth_executor | |
| from gradio_rangeslider import RangeSlider | |
| import logging | |
| logging.basicConfig(level=logging.INFO) | |
| logger = logging.getLogger(__name__) | |
| class UI: | |
| def __init__(self): | |
| """Load any static assets""" | |
| self.load_css() | |
| def header_block(self): | |
| """Title/description""" | |
| with open("assets/header.md", "r") as f: | |
| content = f.read() | |
| gr.Markdown(content) | |
| gr.Markdown("---") | |
| gr.Markdown("<br>") | |
| def selection_panel(self): | |
| """user selections""" | |
| gr.Markdown("""<h1 style='color: purple;'> Ranking with benchmarks </h1> """) | |
| gr.Markdown( | |
| """Using inference data gathered from [HELM](https://crfm.stanford.edu/helm/classic/latest/) we first show how our estimated rankings compare to rankings derived from using ground-truth or reference data.""" | |
| ) | |
| with gr.Column(variant="compact"): | |
| self.data = gr.Dropdown( | |
| choices=["CNN/DM", "XSUM", "MMLU"], | |
| multiselect=False, | |
| value="CNN/DM", | |
| label="Choose a dataset.", | |
| info="The dataset describes a specific task, either summarization (CNN/DM, XSUM) or multiple choice (MMLU).", | |
| interactive=True, | |
| ) | |
| self.mmlu = gr.Dropdown(visible=False) | |
| self.evaluation = gr.Dropdown( | |
| choices=["Rouge", "Equality"], | |
| multiselect=False, | |
| value="Rouge", | |
| interactive=True, | |
| label="Evaluation function", | |
| info="How should the Judge model decide the winner? Demo limited to use 'Rouge' for generative tasks like summarization, and 'equality' for multiple choice or classification tasks. In practice you can use any function that compares judge responses to the contestant models.", | |
| ) | |
| def update_mmlu(v): | |
| if v == "MMLU": | |
| return gr.Dropdown( | |
| choices=list(['abstract_algebra', 'college_chemistry', 'computer_security', 'econometrics', 'us_foreign_policy']), | |
| value='us_foreign_policy', | |
| multiselect=False, | |
| label="Choose MMLU subject.", | |
| info="MMLU subject area.", | |
| interactive=True, | |
| visible=True, | |
| ), gr.Dropdown(choices=['Equality'], value='Equality') | |
| else: | |
| return gr.Dropdown(visible=False), gr.Dropdown(choices=['Rouge'], value='Rouge') | |
| self.data.change(fn=update_mmlu, inputs=self.data, outputs=[self.mmlu, self.evaluation]) | |
| self.nmodels = gr.Dropdown( | |
| choices=["All", 10, 20, 30], | |
| label="Number of models", | |
| info="Sample a subset of LLMs to rank.", | |
| value=10, | |
| interactive=True, | |
| ) | |
| self.nrows = gr.Dropdown( | |
| choices=["All", 10, 20, 30], | |
| label="Number of instances", | |
| info="Sample a subset of instances to evaluate (smaller is faster).", | |
| value=10, | |
| interactive=True, | |
| ) | |
| self.method = gr.Dropdown( | |
| choices=["Greedy", "Full"], | |
| label="Algorithm variant to use", | |
| info="Choose from one of two variants. 'Full' (FTR in the paper) runs all triplet combinations, recommended when evaluations are cheap or for smaller datasets, or 'greedy' (GTR) a faster variant suggested for more complex evaluations.", | |
| value="Full", | |
| interactive=True, | |
| ) | |
| self.btn_execute = gr.Button("Run") | |
| def output_panel(self): | |
| """Plots/leaderboard/bump charts""" | |
| with gr.Column(variant="default"): | |
| gr.Markdown("""<h2 style='color: purple;'> Estimated ranking </h2> """) | |
| self.leaderboard = gr.DataFrame(headers=["rank", "model"], | |
| datatype=["number", "str"]) | |
| with gr.Column(variant="default"): | |
| gr.Markdown( | |
| """<h2 style='color: purple;'> Comparison to 'true' ranking </h2> """ | |
| ) | |
| # self.bumpchart = gr.Plot(format='png') | |
| self.bumpchart = gr.Image() | |
| self.eval_metrics = gr.Markdown() | |
| def synth_panel(self): | |
| """Synthetic data experiments""" | |
| gr.Markdown("<br>") | |
| gr.Markdown("---") | |
| with open("assets/synth.md", "r") as f: | |
| content = f.read() | |
| gr.Markdown(content) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| with gr.Column(variant='compact'): | |
| self.synth_range = RangeSlider(10, 100, value=(50, 90), step=1, label="Model Accuracy Range (%)", interactive=True) | |
| self.synth_nmodels = gr.Slider(3, 50, value=10, step=1, label="Number of models to synthesise.", info="Equally spaced in the accuracy range.", interactive=True) | |
| self.synth_nanswers = gr.Slider(2, 50, value=10, step=1, label="Number of possible (discrete) answers per prompt.", interactive=True) | |
| self.synth_nquestions = gr.Slider(10, 100, step=10, label="Number of prompts to simulate.", interactive=True) | |
| self.synth_noise = gr.Slider(0, 1, value=0, label='Noise in evaluation (p)', info="Evaluation function decisions flipped with probability p. p=0 implies no noise.", interactive=True) | |
| self.synth_method = gr.Dropdown( | |
| choices=["Greedy", "Full"], | |
| label="Algorithm variant to use", | |
| info="Choose from one of two variants. 'Full' (FTR in the paper) runs all triplet combinations, recommended when evaluations are cheap or for smaller datasets, or 'greedy' (GTR) a faster variant suggested for more complex evaluations.", | |
| value="Full", | |
| interactive=True, | |
| ) | |
| examples = gr.Examples([[(10, 30), 10, 10, 10, 0, "Full"], | |
| [(10, 30), 10, 10, 10, 0.5, "Full"], | |
| [[10, 30], 10, 2, 10, 0, "Full"]], | |
| [self.synth_range, self.synth_nmodels, self.synth_nanswers, self.synth_nquestions, self.synth_noise, self.synth_method], | |
| label='Some interesting cases (click and run)', example_labels=["Rankings recovered for low accuracy models", | |
| "Robust recovery when evaluations have noise", | |
| "Binary outcomes are challenging" | |
| ] ) | |
| self.synth_execute = gr.Button("Run") | |
| with gr.Column(scale=1): | |
| with gr.Column(variant="default"): | |
| gr.Markdown( | |
| """<h2 style='color: purple;'> Estimated vs. true ranking </h2> """ | |
| ) | |
| self.synth_bumpchart = gr.Image() | |
| with gr.Column(scale=1): | |
| self.synth_eval_metrics = gr.Markdown() | |
| def byod_panel(self): | |
| """Instructions panel""" | |
| gr.Markdown("<br>") | |
| gr.Markdown("---") | |
| with open("assets/instructions.md", "r") as f: | |
| content = f.read() | |
| gr.Markdown(content) | |
| gr.Markdown("---") | |
| def load_css(self): | |
| with open("style.css", "r") as file: | |
| self.css = file.read() | |
| def layout(self): | |
| """Assemble the overall layout""" | |
| with gr.Blocks(theme=gr.themes.Default()) as demo: | |
| self.header_block() | |
| with gr.Row(): | |
| # Selection panel | |
| with gr.Column(): | |
| self.selection_panel() | |
| # Output panel/leaderboard | |
| self.output_panel() | |
| self.synth_panel() | |
| self.byod_panel() | |
| # Register event listeners | |
| self.btn_execute.click( | |
| fn=benchmark_executor, | |
| inputs=[ | |
| self.data, | |
| self.mmlu, | |
| self.evaluation, | |
| self.nmodels, | |
| self.nrows, | |
| self.method, | |
| ], | |
| outputs=[self.leaderboard, self.bumpchart, self.eval_metrics], | |
| ) | |
| self.synth_execute.click( | |
| fn=synth_executor, | |
| inputs=[ | |
| self.synth_range, | |
| self.synth_nmodels, | |
| self.synth_nanswers, | |
| self.synth_nquestions, | |
| self.synth_noise, | |
| self.synth_method, | |
| ], | |
| outputs=[self.synth_bumpchart, self.synth_eval_metrics], | |
| ) | |
| return demo | |
| def run(self): | |
| self.ui = self.layout() | |
| self.ui.queue().launch(show_error=True) | |
| if __name__ == "__main__": | |
| ui = UI() | |
| ui.run() | |
| #demo = ui.layout() | |
| #demo.launch() | |