app.py

import start
import gradio as gr
import pandas as pd
from glob import glob
from pathlib import Path
from tabs.dashboard import df
from tabs.faq import (
    about_olas_predict_benchmark,
    about_olas_predict,
    about_the_dataset,
    about_the_tools
)
from tabs.howto_benchmark import how_to_run
from tabs.run_benchmark import run_benchmark_main


demo = gr.Blocks()


def run_benchmark_gradio(tool_name, model_name, num_questions, openai_api_key, anthropic_api_key, openrouter_api_key):
    """Run the benchmark using inputs."""
    if tool_name is None:
        return "Please enter the name of your tool."
    if openai_api_key is None and anthropic_api_key is None and openrouter_api_key is None:
        return "Please enter either OpenAI or Anthropic or OpenRouter API key."
    
    result = run_benchmark_main(tool_name, model_name, num_questions, openai_api_key, anthropic_api_key, openrouter_api_key)
    if result == 'completed':
        # get the results file in the results directory
        fns = glob('results/*.csv')

        print(f"Number of files in results directory: {len(fns)}")

        # convert to Path
        files = [Path(file) for file in fns]

        # get results and summary files
        results_files = [file for file in files if 'results' in file.name]

        # the other file is the summary file
        summary_files = [file for file in files if 'summary' in file.name]

        print(results_files, summary_files)

        # get the path with results
        results_df = pd.read_csv(results_files[0])
        summary_df = pd.read_csv(summary_files[0])

        # make sure all df float values are rounded to 4 decimal places
        results_df = results_df.round(4)
        summary_df = summary_df.round(4)

        return gr.Dataframe(value=results_df), gr.Dataframe(value=summary_df)
    
    return gr.Textbox(label="Benchmark Result", value=result, interactive=False), gr.Textbox(label="Summary", value="")


with demo:
    gr.HTML("<h1>Olas Predict Benchmark</hjson>")
    gr.Markdown("Leaderboard showing the performance of Olas Predict tools on the Autocast dataset and overview of the project.")

    with gr.Tabs() as tabs:
        # first tab - leaderboard
        with gr.TabItem("🏅 Benchmark Leaderboard", id=0):

            gr.components.Dataframe(
                value=df,
            )

        # second tab - about
        with gr.TabItem("ℹ️ About"):
            with gr.Row():
                with gr.Accordion("About the Benchmark", open=False):
                    gr.Markdown(about_olas_predict_benchmark)
            with gr.Row():
                with gr.Accordion("About the Tools", open=False):
                    gr.Markdown(about_the_tools)
            with gr.Row():
                with gr.Accordion("About the Autocast Dataset", open=False):
                    gr.Markdown(about_the_dataset)
            with gr.Row():
                with gr.Accordion("About Olas", open=False):
                    gr.Markdown(about_olas_predict)

        
        # third tab - how to run the benchmark
        with gr.TabItem("🚀 Contribute"):
            gr.Markdown(how_to_run)

        # fourth tab - run the benchmark
        with gr.TabItem("🔥 Run the Benchmark"):
            with gr.Row():
                tool_name = gr.Dropdown(
                    [
                        "prediction-offline",
                        "prediction-online",
                        # "prediction-online-summarized-info",
                        # "prediction-offline-sme",
                        # "prediction-online-sme",
                        'prediction-request-rag',
                        'prediction-request-reasoning',
                        # "prediction-url-cot-claude",
                        # "prediction-request-rag-cohere",
                        # "prediction-with-research-conservative",
                        # "prediction-with-research-bold",
                    ], label="Tool Name", info="Choose the tool to run")
                model_name = gr.Dropdown([
                    "gpt-3.5-turbo-0125",
                    "gpt-4-0125-preview",
                    "claude-3-haiku-20240307", 
                    "claude-3-sonnet-20240229", 
                    "claude-3-opus-20240229",
                    "databricks/dbrx-instruct:nitro",
                    "nousresearch/nous-hermes-2-mixtral-8x7b-sft",
                    # "cohere/command-r-plus",
                ], label="Model Name", info="Choose the model to use")
            with gr.Row():
                openai_api_key = gr.Textbox(label="OpenAI API Key", placeholder="Enter your OpenAI API key here", type="password")
                anthropic_api_key = gr.Textbox(label="Anthropic API Key", placeholder="Enter your Anthropic API key here", type="password")
                openrouter_api_key = gr.Textbox(label="OpenRouter API Key", placeholder="Enter your OpenRouter API key here", type="password")
            with gr.Row():
                num_questions = gr.Slider(
                                    minimum=1,
                                    maximum=340,
                                    value=10,
                                    label="Number of questions to run the benchmark on",
                                )
            with gr.Row():
                run_button = gr.Button("Run Benchmark")
            with gr.Row():
                with gr.Accordion("Results", open=True):
                    result = gr.Dataframe()
            with gr.Row():
                with gr.Accordion("Summary", open=False):
                    summary = gr.Dataframe()
            
            run_button.click(run_benchmark_gradio, 
                            inputs=[tool_name, model_name, num_questions, openai_api_key, anthropic_api_key, openrouter_api_key], 
                            outputs=[result, summary])


demo.queue(default_concurrency_limit=40).launch()