fix submodule

.gitignore

@@ -1 +1,165 @@
-__pycache__
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+flagged/
+results/
+uploads/
+__pycache__/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "olas-predict-benchmark"]
+	path = olas-predict-benchmark
+	url = https://github.com/valory-xyz/olas-predict-benchmark.git

README.md

@@ -1,10 +1,10 @@
 ---
-title: Olas Prediction Leaderboard
-emoji: 📊
-colorFrom: red
-colorTo: green
+title: Leaderboard Gradio
+emoji: 🦀
+colorFrom: pink
+colorTo: blue
 sdk: gradio
-sdk_version: 4.23.0
+sdk_version: 4.25.0
 app_file: app.py
 pinned: false
 license: apache-2.0

app.py

@@ -1,61 +1,145 @@
+import start
 import gradio as gr
 import pandas as pd
+from glob import glob
+from pathlib import Path
+from tabs.dashboard import df
+from tabs.faq import (
+    about_olas_predict_benchmark,
+    about_olas_predict,
+    about_the_dataset,
+    about_the_tools
+)
+from tabs.howto_benchmark import how_to_run
+from tabs.run_benchmark import run_benchmark_main
 
 
+demo = gr.Blocks()
 
-csv_file_path = "formatted_data.csv"
-
-# Reading the CSV file
-df = pd.read_csv(csv_file_path)
-
-# Markdown text with HTML formatting for Gradio
-markdown_text = """
-## Benchmark Overview
-- The benchmark evaluates the performance of Olas Predict tools on the Autocast dataset.
-- The dataset has been refined to enhance the evaluation of the tools.
-- The leaderboard shows the performance of the tools based on the refined dataset.
-- The script to run the benchmark is available in the repo [here](https://github.com/valory-xyz/olas-predict-benchmark).
-
-## How to run your tools on the benchmark
-- Fork the repo [here](https://github.com/valory-xyz/olas-predict-benchmark).
-- Git init the submodules and update the submodule to get the latest dataset `mech` tool.
-    - `git submodule init`
-    - `git submodule update --remote --recursive`
-- Include your tool in the `mech/packages` directory accordingly.
-    - Guidelines on how to include your tool can be found [here](xxx).
-- Run the benchmark script.
-
-## Dataset Overview
-This project leverages the Autocast dataset from the research paper titled ["Forecasting Future World Events with Neural Networks"](https://arxiv.org/abs/2206.15474). 
-The dataset has undergone further refinement to enhance the performance evaluation of Olas mech prediction tools. 
-Both the original and refined datasets are hosted on HuggingFace.
-
-### Refined Dataset Files
-- You can find the refined dataset on HuggingFace [here](https://huggingface.co/datasets/valory/autocast).
-- `autocast_questions_filtered.json`: A JSON subset of the initial autocast dataset.
-- `autocast_questions_filtered.pkl`: A pickle file mapping URLs to their respective scraped documents within the filtered dataset.
-- `retrieved_docs.pkl`: Contains all the scraped texts.
-
-### Filtering Criteria
-To refine the dataset, we applied the following criteria to ensure the reliability of the URLs:
-- URLs not returning HTTP 200 status codes are excluded.
-- Difficult-to-scrape sites, such as Twitter and Bloomberg, are omitted.
-- Links with less than 1000 words are removed.
-- Only samples with a minimum of 5 and a maximum of 20 working URLs are retained.
-
-### Scraping Approach
-The content of the filtered URLs has been scraped using various libraries, depending on the source:
-- `pypdf2` for PDF URLs.
-- `wikipediaapi` for Wikipedia pages.
-- `requests`, `readability-lxml`, and `html2text` for most other sources.
-- `requests`, `beautifulsoup`, and `html2text` for BBC links.
-"""
-
-
-with gr.Blocks() as demo:
-    gr.Markdown("# Olas Predict Benchmark")
+
+def run_benchmark_gradio(tool_name, model_name, num_questions, openai_api_key, anthropic_api_key):
+    """Run the benchmark using inputs."""
+    if tool_name is None:
+        return "Please enter the name of your tool."
+    if openai_api_key is None and anthropic_api_key is None:
+        return "Please enter either OpenAI or Anthropic API key."
+    
+    result = run_benchmark_main(tool_name, model_name, num_questions, openai_api_key, anthropic_api_key)
+    if result == 'completed':
+        # get the results file in the results directory
+        fns = glob('results/*.csv')
+
+        print(f"Number of files in results directory: {len(fns)}")
+
+        # convert to Path
+        files = [Path(file) for file in fns]
+
+        # get results and summary files
+        results_files = [file for file in files if 'results' in file.name]
+
+        # the other file is the summary file
+        summary_files = [file for file in files if 'summary' in file.name]
+
+        print(results_files, summary_files)
+
+        # get the path with results
+        results_df = pd.read_csv(results_files[0])
+        summary_df = pd.read_csv(summary_files[0])
+
+        # make sure all df float values are rounded to 4 decimal places
+        results_df = results_df.round(4)
+        summary_df = summary_df.round(4)
+
+        return gr.Dataframe(value=results_df), gr.Dataframe(value=summary_df)
+    
+    return gr.Textbox(label="Benchmark Result", value=result, interactive=False), gr.Textbox(label="Summary", value="")
+
+
+with demo:
+    gr.HTML("<h1>Olas Predict Benchmark</hjson>")
     gr.Markdown("Leaderboard showing the performance of Olas Predict tools on the Autocast dataset and overview of the project.")
-    gr.DataFrame(df)
-    gr.Markdown(markdown_text)
 
-demo.launch(debug=True)
+    with gr.Tabs() as tabs:
+        # first tab - leaderboard
+        with gr.TabItem("🏅 Benchmark Leaderboard", id=0):
+
+            gr.components.Dataframe(
+                value=df,
+            )
+
+        # second tab - about
+        with gr.TabItem("ℹ️ About"):
+            with gr.Row():
+                with gr.Accordion("About the Benchmark", open=False):
+                    gr.Markdown(about_olas_predict_benchmark)
+            with gr.Row():
+                with gr.Accordion("About the Tools", open=False):
+                    gr.Markdown(about_the_tools)
+            with gr.Row():
+                with gr.Accordion("About the Autocast Dataset", open=False):
+                    gr.Markdown(about_the_dataset)
+            with gr.Row():
+                with gr.Accordion("About Olas", open=False):
+                    gr.Markdown(about_olas_predict)
+
+        
+        # third tab - how to run the benchmark
+        with gr.TabItem("🚀 Contribute"):
+            gr.Markdown(how_to_run)
+
+        def update_dropdown(tool):
+            if "claude" in tool:
+                return ["claude-3-haiku-20240307", "claude-3-sonnet-20240229", "claude-3-opus-20240229"]
+            else:
+                return ["gpt-3.5-turbo-0125", "gpt-4-0125-preview"]
+
+
+        # fourth tab - run the benchmark
+        with gr.TabItem("🔥 Run the Benchmark"):
+            with gr.Row():
+                tool_name = gr.Dropdown(
+                    [
+                        "prediction-offline",
+                        "prediction-online",
+                        "prediction-offline-sme",
+                        "prediction-online-sme",
+                        "claude-prediction-offline",
+                        "claude-prediction-online",
+                        'prediction-request-rag',
+                        "prediction-with-research-conservative",
+                        "prediction-with-research-bold",
+                        "prediction-request-reasoning-claude",
+                        "prediction-request-rag-claude",
+                        "prediction-url-cot-claude",
+                    ], label="Tool Name", info="Choose the tool to run")
+                model_name = gr.Dropdown([
+                    "gpt-3.5-turbo-0125",
+                    "gpt-4-0125-preview"
+                    "claude-3-haiku-20240307", 
+                    "claude-3-sonnet-20240229", 
+                    "claude-3-opus-20240229",
+                ], label="Model Name", info="Choose the model to use")
+            with gr.Row():
+                openai_api_key = gr.Textbox(label="OpenAI API Key", placeholder="Enter your OpenAI API key here", type="password")
+                anthropic_api_key = gr.Textbox(label="Anthropic API Key", placeholder="Enter your Anthropic API key here", type="password")
+            with gr.Row():
+                num_questions = gr.Slider(
+                                    minimum=1,
+                                    maximum=340,
+                                    value=10,
+                                    label="Number of questions to run the benchmark on",
+                                )
+            with gr.Row():
+                run_button = gr.Button("Run Benchmark")
+            with gr.Row():
+                with gr.Accordion("Results", open=True):
+                    result = gr.Dataframe()
+            with gr.Row():
+                with gr.Accordion("Summary", open=False):
+                    summary = gr.Dataframe()
+            
+            run_button.click(run_benchmark_gradio, 
+                            inputs=[tool_name, model_name, num_questions, openai_api_key, anthropic_api_key], 
+                            outputs=[result, summary])
+
+demo.queue(default_concurrency_limit=40).launch()

formatted_data.csv

@@ -1,8 +1,23 @@
-Tool,Accuracy,Correct,Total,Mean Tokens Used,Mean Cost ($)
-claude-prediction-offline,0.7201834862385321,157,218,779.4770642201835,0.006891669724770637
-claude-prediction-online,0.6600660066006601,200,303,1505.3135313531352,0.013348171617161701
-prediction-online,0.676737160120846,224,331,1219.6918429003022,0.001332990936555879
-prediction-offline,0.6599326599326599,196,297,579.6565656565657,0.000621023569023569
-prediction-online-summarized-info,0.6209150326797386,190,306,1008.4542483660131,0.0011213790849673195
-prediction-offline-sme,0.599406528189911,202,337,1190.2017804154302,0.0013518635014836643
-prediction-online-sme,0.5905044510385756,199,337,1834.919881305638,0.0020690207715133428
+Tool,Model,Accuracy,Correct,Total,Mean Tokens Used,Mean Cost ($)
+claude-prediction-offline,claude-3-sonnet-20240229,0.756838905775076,249,329,920.5987841945289,0.0033739787234042555
+claude-prediction-offline,claude-3-opus-20240229,0.7558823529411764,257,340,920.45,0.016866044117647028
+prediction-request-reasoning-claude,claude-3-sonnet-20240229,0.753125,241,320,2645.509375,0.019254515624999982
+prediction-offline,gpt-4-0125-preview,0.7507692307692307,244,325,727.1846153846154,0.008048953846153844
+prediction-offline-sme,gpt-4-0125-preview,0.7484848484848485,247,330,1416.8484848484848,0.018169212121212114
+prediction-request-reasoning,gpt-4-0125-preview,0.7483221476510067,223,298,1980.7281879194632,0.02567674496644293
+claude-prediction-online,claude-3-sonnet-20240229,0.7411764705882353,252,340,2832.7617647058823,0.00959039117647058
+prediction-url-cot-claude,claude-3-sonnet-20240229,0.7355623100303952,242,329,14789.27963525836,0.0510609574468085
+prediction-request-reasoning-claude,claude-3-opus-20240229,0.7337278106508875,248,338,2773.284023668639,0.10624464497041416
+prediction-request-rag-claude,claude-3-sonnet-20240229,0.7331288343558282,239,326,2850.1196319018404,0.01465865337423311
+claude-prediction-offline,claude-2,0.7201834862385321,157,218,779.4770642201835,0.006891669724770637
+prediction-request-rag,gpt-4-0125-preview,0.7161716171617162,217,303,1240.980198019802,0.013809207920792065
+prediction-request-reasoning-claude,claude-3-haiku-20240307,0.6982248520710059,236,338,2700.6508875739646,0.0016877189349112328
+prediction-with-research-bold,gpt-4-1106-preview,0.6938775510204082,34,49,9319.244897959185,0.11741489795918365
+prediction-online,gpt-4-0125-preview,0.713855421686747,237,332,1549.8524096385543,0.017273584337349383
+prediction-online-sme,gpt-4-0125-preview,0.7012195121951219,230,328,2237.868902439024,0.027385884146341445
+claude-prediction-online,claude-2,0.6600660066006601,200,303,1505.3135313531352,0.013348171617161701
+prediction-offline,gpt-3.5-turbo-0125,0.6578171091445427,223,339,730.1740412979351,0.0007721681415928988
+prediction-request-reasoning,gpt-3.5-turbo-0125,0.6506410256410257,203,312,1871.173076923077,0.002112727564102551
+prediction-offline-sme,gpt-3.5-turbo-0125,0.6294117647058823,214,340,1341.8323529411764,0.0014778852941176408
+prediction-online,gpt-3.5-turbo-0125,0.551622418879056,187,339,1576.684365781711,0.0016928525073746164
+prediction-online-sme,gpt-3.5-turbo-0125,0.49411764705882355,168,340,2189.1882352941175,0.002402523529411752

olas-predict-benchmark

@@ -0,0 +1 @@
+Subproject commit 04f6cf738bc2fcae0416761ce3648f6dfecd08fc

requirements.txt

@@ -1 +0,0 @@
-gradio

start.py

@@ -0,0 +1,66 @@
+import os
+import sys
+import subprocess
+from huggingface_hub import hf_hub_download
+
+
+def run_command(command: str, cwd: str = None) -> tuple:
+    """Run a shell command in the specified directory and return the output."""
+    process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
+    stdout, stderr = process.communicate()
+    if process.returncode != 0:
+        print(f"Error: {stderr.decode()}")
+    else:
+        print(f"Output: {stdout.decode()}")
+    return stdout, stderr
+
+def download_dataset():
+    """Download the dataset."""
+    print("Downloading the dataset...")
+    repo_id = "valory/autocast"
+    base_dir = os.getcwd()
+    output_dir = os.path.join(base_dir, "olas-predict-benchmark", "benchmark", "data", "autocast")
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir, exist_ok=True)
+    filenames = [
+        "autocast_questions_filtered.json",
+        "autocast_questions_filtered.pkl",
+    ]
+    for filename in filenames:
+        hf_hub_download(repo_id=repo_id, filename=filename, local_dir=output_dir, repo_type="dataset")
+    print("Dataset downloaded successfully.")
+
+def start():
+    """Start commands."""
+    print("Starting commands...")
+    base_dir = os.getcwd()
+    olas_dir = os.path.join(base_dir, "olas-predict-benchmark")
+    mech_dir = os.path.join(olas_dir, "benchmark", "mech")
+
+    commands = [
+        ("git submodule init", base_dir),
+        ("git submodule update --init --recursive", base_dir),
+        ("git submodule update --remote --recursive", base_dir),
+        ('git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', olas_dir),
+        ("git remote update", olas_dir),
+        ("git fetch --all", olas_dir),
+        ("git show-ref --verify --quiet \"refs/remotes/origin/fix/mech-packages\"", olas_dir),
+        ("git checkout fix/mech-packages", olas_dir),
+        ("git pull origin fix/mech-packages", olas_dir),
+        ("git checkout main", mech_dir),
+        ("git pull origin main", mech_dir),
+        ("pip install -e .", os.path.join(olas_dir, "benchmark")),
+        ("pip install -e .", mech_dir),
+        ("pip install lxml[html_clean]", base_dir),
+    ]
+
+    for command, cwd in commands:
+        run_command(command, cwd=cwd)
+
+    # add benchmark to the path
+    sys.path.append(os.path.join(olas_dir, "benchmark"))
+
+    # Download the dataset
+    download_dataset()
+
+start()

tabs/dashboard.py

@@ -0,0 +1,15 @@
+import gradio as gr
+import pandas as pd
+
+csv_file_path = "formatted_data.csv"
+
+def return_df():
+    # Reading the CSV file
+    df = pd.read_csv(csv_file_path)
+
+    # all floats to be rounded to 2 decimal places
+    df = df.round(4)
+    return df
+
+
+df = return_df()

tabs/faq.py

@@ -0,0 +1,48 @@
+about_olas_predict_benchmark = """\
+How good are LLMs at making predictions about events in the future? This is a topic that hasn't been well explored to date. 
+[Olas Predict](https://olas.network/services/prediction-agents) aims to rectify this by incentivizing the creation of agents that predict the future (through prediction markets).
+This is a leaderboard showing the performance of LLM tools for making predictions (event forecasting) on a dataset, refined from Autocast.\
+The leaderboard shows tool performance in terms of accuracy and cost. \
+
+🤗 Pick a tool and run it on the benchmark using the "🔥 Run the Benchmark" page! 
+"""
+
+about_the_tools = """\
+- [Prediction Offline](https://github.com/valory-xyz/mech/blob/main/packages/valory/customs/prediction_request/prediction_request.py) - Uses prompt engineering, but no web crawling, to make predictions
+- [Prediction Online](https://github.com/valory-xyz/mech/blob/main/packages/valory/customs/prediction_request/prediction_request.py) - Uses prompt engineering, as well as web crawling, to make predictions
+- [Prediction SME](https://github.com/valory-xyz/mech/blob/main/packages/nickcom007/customs/prediction_request_sme/prediction_request_sme.py) - Use prompt engineering to get the LLM to act as a Subject Matter Expert (SME) in making a prediction.
+- [Prediction with RAG](https://github.com/valory-xyz/mech/blob/main/packages/napthaai/customs/prediction_request_rag/prediction_request_rag.py) - Uses retrieval-augment-generation (RAG) over extracted search result to make predictions.
+- [Prediction with Research Report](https://github.com/valory-xyz/mech/blob/main/packages/polywrap/customs/prediction_with_research_report/prediction_with_research_report.py) - Generates a research report before making a prediction.
+- [Prediction with Reasoning](https://github.com/valory-xyz/mech/blob/main/packages/napthaai/customs/prediction_request_reasoning/prediction_request_reasoning.py) - Incorporates an additional call to the LLM to do reasoning over retrieved data.
+- [Prediction with CoT](https://github.com/valory-xyz/mech/blob/main/packages/napthaai/customs/prediction_url_cot/prediction_url_cot.py) - Use Chain of Thought (CoT) to make predictions.
+"""
+
+about_the_dataset = """\
+## Dataset Overview
+This project leverages the Autocast dataset from the research paper titled ["Forecasting Future World Events with Neural Networks"](https://arxiv.org/abs/2206.15474). 
+The dataset has undergone further refinement to enhance the performance evaluation of Olas mech prediction tools. 
+Both the original and refined datasets are hosted on HuggingFace.
+### Refined Dataset Files
+- You can find the refined dataset on HuggingFace [here](https://huggingface.co/datasets/valory/autocast).
+- `autocast_questions_filtered.json`: A JSON subset of the initial autocast dataset.
+- `autocast_questions_filtered.pkl`: A pickle file mapping URLs to their respective scraped documents within the filtered dataset.
+- `retrieved_docs.pkl`: Contains all the scraped texts.
+### Filtering Criteria
+To refine the dataset, we applied the following criteria to ensure the reliability of the URLs:
+- URLs not returning HTTP 200 status codes are excluded.
+- Difficult-to-scrape sites, such as Twitter and Bloomberg, are omitted.
+- Links with less than 1000 words are removed.
+- Only samples with a minimum of 5 and a maximum of 20 working URLs are retained.
+### Scraping Approach
+The content of the filtered URLs has been scraped using various libraries, depending on the source:
+- `pypdf2` for PDF URLs.
+- `wikipediaapi` for Wikipedia pages.
+- `requests`, `readability-lxml`, and `html2text` for most other sources.
+- `requests`, `beautifulsoup`, and `html2text` for BBC links.
+"""
+
+about_olas_predict = """\
+Olas is a network of autonomous services that can run complex logic in a decentralized manner, interacting with on- and off-chain data autonomously and continuously. For other use cases check out [olas.network](https://olas.network/). 
+Since 'Olas' means 'waves' in Spanish, it is sometimes referred to as the 'ocean of services' 🌊.
+The project is co-created by [Valory](https://www.valory.xyz/). Valory aspires to enable communities, organizations and countries to co-own AI systems, beginning with decentralized autonomous agents.
+"""

tabs/howto_benchmark.py

@@ -0,0 +1,26 @@
+how_to_run = """\
+## Create a new Tool
+
+- Fork the repo [here](https://github.com/valory-xyz/mech).
+- Create your tool in mech/packages/{your-name-here}/f'{your-tool}.py'
+- Submit a PR
+
+## Test your Tool on the Benchmark On HF
+
+- Navigate to the "🔥 Run the Benchmark" page
+- Choose your tool and model
+- Enter your API key
+- Press "Run Benchmark"
+- Inspect the results
+
+## Test your Tool on the Benchmark locally
+
+- Fork the repo [here](https://github.com/valory-xyz/olas-predict-benchmark).
+- Follow the instructions in the readme.
+
+## Run a Prediction Agent that uses these Tools to place real bets
+
+- Fork the repo [here](https://github.com/valory-xyz/trader-quickstart).
+- Follow the instructions in the readme.
+
+"""

tabs/run_benchmark.py

@@ -0,0 +1,37 @@
+import os
+from benchmark.run_benchmark import run_benchmark
+
+
+def run_benchmark_main(tool_name, model_name, num_questions, openai_api_key, anthropic_api_key):
+    """Run the benchmark using the provided function and API key."""
+    # Empyt the results directory
+    os.system("rm -rf results/*")
+
+    # Set the benchmark parameters
+    kwargs = {}
+    if not num_questions:
+        num_questions = 10
+    kwargs["num_questions"] = num_questions
+    kwargs["tools"] = [tool_name]
+    if model_name:
+        kwargs["model"] = model_name
+    kwargs["api_keys"] = {}
+    if openai_api_key:
+        kwargs["api_keys"]["openai"] = openai_api_key
+    if anthropic_api_key:
+        kwargs["api_keys"]["anthropic"] = anthropic_api_key
+
+    kwargs["num_urls"] = 3
+    kwargs["num_words"] = 300
+    kwargs["provide_source_links"] = True
+
+    print(f"Running benchmark with the following parameters: {kwargs}")
+    
+    # Run the benchmark
+    try:
+        run_benchmark(kwargs=kwargs)
+        return "completed"
+    except Exception as e:
+        return f"Error running benchmark: {e}"
+
+