initial commit

.gitignore

@@ -0,0 +1,165 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+flagged/
+results/
+uploads/
+__pycache__/

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "olas-predict-benchmark"]
+	path = olas-predict-benchmark
+	url = https://github.com/valory-xyz/olas-predict-benchmark.git
+	branch = fix/mech-packages

Dockerfile

@@ -0,0 +1,30 @@
+# Use an official Python runtime as a parent image
+FROM python:3.10-slim-buster
+
+# Set the working directory in the container to /app
+WORKDIR /app
+
+# Copy the current directory contents into the container at /app
+COPY . /app
+
+# Install git and any other dependencies you might need
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install Poetry
+RUN pip install poetry
+
+
+# Run submodule update and checkout commands
+RUN git submodule update --init --recursive && \
+    cd olas-predict-benchmark/benchmark && git checkout fix/mech-packages && cd ../.. && \
+    cd olas-predict-benchmark/benchmark/mech && git checkout main && cd ../../..
+
+# Install project dependencies
+RUN poetry install --no-interaction --no-ansi
+
+EXPOSE 7860
+
+CMD ["poetry", "run", "python", "app.py"]

app.py

@@ -0,0 +1,139 @@
+import gradio as gr
+import pandas as pd
+from glob import glob
+from pathlib import Path
+from tabs.dashboard import df
+from tabs.faq import (
+    about_olas_predict_benchmark,
+    about_olas_predict,
+    about_the_dataset,
+    about_the_tools
+)
+
+from tabs.howto_benchmark import how_to_run
+from tabs.run_benchmark import run_benchmark_main
+
+
+demo = gr.Blocks()
+
+
+def run_benchmark_gradio(tool_name, model_name, openai_api_key, anthropic_api_key):
+    """Run the benchmark using inputs."""
+    if tool_name is None:
+        return "Please enter the name of your tool."
+    if openai_api_key is None and anthropic_api_key is None:
+        return "Please enter either OpenAI or Anthropic API key."
+    
+    result = run_benchmark_main(tool_name, model_name, openai_api_key, anthropic_api_key)
+    if result == 'completed':
+        # get the results file in the results directory
+        fns = glob('results/*.csv')
+
+        print(f"Number of files in results directory: {len(fns)}")
+
+        # convert to Path
+        files = [Path(file) for file in fns]
+
+        # get results and summary files
+        results_files = [file for file in files if 'results' in file.name]
+
+        # the other file is the summary file
+        summary_files = [file for file in files if 'summary' in file.name]
+
+        print(results_files, summary_files)
+
+        # get the path with results
+        results_df = pd.read_csv(results_files[0])
+        summary_df = pd.read_csv(summary_files[0])
+
+        # make sure all df float values are rounded to 4 decimal places
+        results_df = results_df.round(4)
+        summary_df = summary_df.round(4)
+
+        return gr.Dataframe(value=results_df), gr.Dataframe(value=summary_df)
+    
+    return gr.Textbox(label="Benchmark Result", value=result, interactive=False), gr.Textbox(label="Summary", value="")
+
+
+with demo:
+    gr.HTML("<h1>Olas Predict Benchmark</hjson>")
+    gr.Markdown("Leaderboard showing the performance of Olas Predict tools on the Autocast dataset and overview of the project.")
+
+    with gr.Tabs() as tabs:
+        # first tab - leaderboard
+        with gr.TabItem("🏅 Benchmark Leaderboard", id=0):
+
+            gr.components.Dataframe(
+                value=df,
+            )
+
+        # second tab - about
+        with gr.TabItem("ℹ️ About"):
+            with gr.Row():
+                with gr.Accordion("About the Benchmark", open=False):
+                    gr.Markdown(about_olas_predict_benchmark)
+            with gr.Row():
+                with gr.Accordion("About the Tools", open=False):
+                    gr.Markdown(about_the_tools)
+            with gr.Row():
+                with gr.Accordion("About the Autocast Dataset", open=False):
+                    gr.Markdown(about_the_dataset)
+            with gr.Row():
+                with gr.Accordion("About Olas", open=False):
+                    gr.Markdown(about_olas_predict)
+
+        
+        # third tab - how to run the benchmark
+        with gr.TabItem("🚀 Contribute"):
+            gr.Markdown(how_to_run)
+
+        def update_dropdown(tool):
+            if "claude" in tool:
+                return ["claude-3-haiku-20240307", "claude-3-sonnet-20240229", "claude-3-opus-20240229"]
+            else:
+                return ["gpt-3.5-turbo-0125", "gpt-4-0125-preview"]
+
+
+        # fourth tab - run the benchmark
+        with gr.TabItem("🔥 Run the Benchmark"):
+            with gr.Row():
+                tool_name = gr.Dropdown(
+                    [
+                        "prediction-offline",
+                        "prediction-online",
+                        # "prediction-online-summarized-info",
+                        "prediction-offline-sme",
+                        "prediction-online-sme",
+                        "claude-prediction-offline",
+                        "claude-prediction-online",
+                        'prediction-request-rag',
+                        "prediction-with-research-conservative",
+                        "prediction-with-research-bold",
+                        "prediction-request-reasoning-claude",
+                        "prediction-request-rag-claude",
+                        "prediction-url-cot-claude",
+                    ], label="Tool Name", info="Choose the tool to run")
+                model_name = gr.Dropdown([
+                    "gpt-3.5-turbo-0125",
+                    "gpt-4-0125-preview"
+                    "claude-3-haiku-20240307", 
+                    "claude-3-sonnet-20240229", 
+                    "claude-3-opus-20240229",
+                ], label="Model Name", info="Choose the model to use")
+            with gr.Row():
+                openai_api_key = gr.Textbox(label="OpenAI API Key", placeholder="Enter your OpenAI API key here", type="password")
+                anthropic_api_key = gr.Textbox(label="Anthropic API Key", placeholder="Enter your Anthropic API key here", type="password")
+            with gr.Row():
+                run_button = gr.Button("Run Benchmark")
+            with gr.Row():
+                with gr.Accordion("Results", open=True):
+                    result = gr.Dataframe()
+            with gr.Row():
+                with gr.Accordion("Summary", open=False):
+                    summary = gr.Dataframe()
+            
+            run_button.click(run_benchmark_gradio, 
+                            inputs=[tool_name, model_name, openai_api_key, anthropic_api_key], 
+                            outputs=[result, summary])
+
+demo.queue(default_concurrency_limit=40).launch(server_port=7860)

formatted_data.csv

@@ -0,0 +1,23 @@
+Tool,Model,Accuracy,Correct,Total,Mean Tokens Used,Mean Cost ($)
+claude-prediction-offline,claude-3-sonnet-20240229,0.756838905775076,249,329,920.5987841945289,0.0033739787234042555
+claude-prediction-offline,claude-3-opus-20240229,0.7558823529411764,257,340,920.45,0.016866044117647028
+prediction-request-reasoning-claude,claude-3-sonnet-20240229,0.753125,241,320,2645.509375,0.019254515624999982
+prediction-offline,gpt-4-0125-preview,0.7507692307692307,244,325,727.1846153846154,0.008048953846153844
+prediction-offline-sme,gpt-4-0125-preview,0.7484848484848485,247,330,1416.8484848484848,0.018169212121212114
+prediction-request-reasoning,gpt-4-0125-preview,0.7483221476510067,223,298,1980.7281879194632,0.02567674496644293
+claude-prediction-online,claude-3-sonnet-20240229,0.7411764705882353,252,340,2832.7617647058823,0.00959039117647058
+prediction-url-cot-claude,claude-3-sonnet-20240229,0.7355623100303952,242,329,14789.27963525836,0.0510609574468085
+prediction-request-reasoning-claude,claude-3-opus-20240229,0.7337278106508875,248,338,2773.284023668639,0.10624464497041416
+prediction-request-rag-claude,claude-3-sonnet-20240229,0.7331288343558282,239,326,2850.1196319018404,0.01465865337423311
+claude-prediction-offline,claude-2,0.7201834862385321,157,218,779.4770642201835,0.006891669724770637
+prediction-request-rag,gpt-4-0125-preview,0.7161716171617162,217,303,1240.980198019802,0.013809207920792065
+prediction-request-reasoning-claude,claude-3-haiku-20240307,0.6982248520710059,236,338,2700.6508875739646,0.0016877189349112328
+prediction-with-research-bold,gpt-4-1106-preview,0.6938775510204082,34,49,9319.244897959185,0.11741489795918365
+prediction-online,gpt-4-0125-preview,0.713855421686747,237,332,1549.8524096385543,0.017273584337349383
+prediction-online-sme,gpt-4-0125-preview,0.7012195121951219,230,328,2237.868902439024,0.027385884146341445
+claude-prediction-online,claude-2,0.6600660066006601,200,303,1505.3135313531352,0.013348171617161701
+prediction-offline,gpt-3.5-turbo-0125,0.6578171091445427,223,339,730.1740412979351,0.0007721681415928988
+prediction-request-reasoning,gpt-3.5-turbo-0125,0.6506410256410257,203,312,1871.173076923077,0.002112727564102551
+prediction-offline-sme,gpt-3.5-turbo-0125,0.6294117647058823,214,340,1341.8323529411764,0.0014778852941176408
+prediction-online,gpt-3.5-turbo-0125,0.551622418879056,187,339,1576.684365781711,0.0016928525073746164
+prediction-online-sme,gpt-3.5-turbo-0125,0.49411764705882355,168,340,2189.1882352941175,0.002402523529411752

pyproject.toml

@@ -0,0 +1,17 @@
+[tool.poetry]
+name = "leaderboard-gradio"
+version = "0.1.0"
+description = ""
+authors = ["Richard Blythman <richardblythman@gmail.com>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+benchmark = { path = "./olas-predict-benchmark/benchmark", develop = true }
+mech = { path = "./olas-predict-benchmark/benchmark/mech", develop = true }
+gradio = "^4.25.0"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

start.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Update and initialize submodules recursively
+git submodule update --init --recursive
+
+# Checkout specific branches for your submodules
+cd olas-predict-benchmark/benchmark
+git checkout fix/mech-packages
+cd ../..
+
+cd olas-predict-benchmark/benchmark/mech
+git checkout main
+cd ../../..
+
+# Configure poetry to create virtual environments within the project directory
+poetry config virtualenvs.in-project true
+
+# Install dependencies as specified in poetry.lock file
+poetry install
+
+# run gradio using the command below
+poetry run python app.py

tabs/dashboard.py

@@ -0,0 +1,15 @@
+import gradio as gr
+import pandas as pd
+
+csv_file_path = "formatted_data.csv"
+
+def return_df():
+    # Reading the CSV file
+    df = pd.read_csv(csv_file_path)
+
+    # all floats to be rounded to 2 decimal places
+    df = df.round(4)
+    return df
+
+
+df = return_df()

tabs/faq.py

@@ -0,0 +1,48 @@
+about_olas_predict_benchmark = """\
+How good are LLMs at making predictions about events in the future? This is a topic that hasn't been well explored to date. 
+[Olas Predict](https://olas.network/services/prediction-agents) aims to rectify this by incentivizing the creation of agents that predict the future (through prediction markets).
+This is a leaderboard showing the performance of LLM tools for making predictions (event forecasting) on a dataset, refined from Autocast.\
+The leaderboard shows tool performance in terms of accuracy and cost. \
+
+🤗 Pick a tool and run it on the benchmark using the "🔥 Run the Benchmark" page! 
+"""
+
+about_the_tools = """\
+- [Prediction Offline](https://github.com/valory-xyz/mech/blob/main/packages/valory/customs/prediction_request/prediction_request.py) - Uses prompt engineering, but no web crawling, to make predictions
+- [Prediction Online](https://github.com/valory-xyz/mech/blob/main/packages/valory/customs/prediction_request/prediction_request.py) - Uses prompt engineering, as well as web crawling, to make predictions
+- [Prediction SME](https://github.com/valory-xyz/mech/blob/main/packages/nickcom007/customs/prediction_request_sme/prediction_request_sme.py) - Use prompt engineering to get the LLM to act as a Subject Matter Expert (SME) in making a prediction.
+- [Prediction with RAG](https://github.com/valory-xyz/mech/blob/main/packages/napthaai/customs/prediction_request_rag/prediction_request_rag.py) - Uses retrieval-augment-generation (RAG) over extracted search result to make predictions.
+- [Prediction with Research Report](https://github.com/valory-xyz/mech/blob/main/packages/polywrap/customs/prediction_with_research_report/prediction_with_research_report.py) - Generates a research report before making a prediction.
+- [Prediction with Reasoning](https://github.com/valory-xyz/mech/blob/main/packages/napthaai/customs/prediction_request_reasoning/prediction_request_reasoning.py) - Incorporates an additional call to the LLM to do reasoning over retrieved data.
+- [Prediction with CoT](https://github.com/valory-xyz/mech/blob/main/packages/napthaai/customs/prediction_url_cot/prediction_url_cot.py) - Use Chain of Thought (CoT) to make predictions.
+"""
+
+about_the_dataset = """\
+## Dataset Overview
+This project leverages the Autocast dataset from the research paper titled ["Forecasting Future World Events with Neural Networks"](https://arxiv.org/abs/2206.15474). 
+The dataset has undergone further refinement to enhance the performance evaluation of Olas mech prediction tools. 
+Both the original and refined datasets are hosted on HuggingFace.
+### Refined Dataset Files
+- You can find the refined dataset on HuggingFace [here](https://huggingface.co/datasets/valory/autocast).
+- `autocast_questions_filtered.json`: A JSON subset of the initial autocast dataset.
+- `autocast_questions_filtered.pkl`: A pickle file mapping URLs to their respective scraped documents within the filtered dataset.
+- `retrieved_docs.pkl`: Contains all the scraped texts.
+### Filtering Criteria
+To refine the dataset, we applied the following criteria to ensure the reliability of the URLs:
+- URLs not returning HTTP 200 status codes are excluded.
+- Difficult-to-scrape sites, such as Twitter and Bloomberg, are omitted.
+- Links with less than 1000 words are removed.
+- Only samples with a minimum of 5 and a maximum of 20 working URLs are retained.
+### Scraping Approach
+The content of the filtered URLs has been scraped using various libraries, depending on the source:
+- `pypdf2` for PDF URLs.
+- `wikipediaapi` for Wikipedia pages.
+- `requests`, `readability-lxml`, and `html2text` for most other sources.
+- `requests`, `beautifulsoup`, and `html2text` for BBC links.
+"""
+
+about_olas_predict = """\
+Olas is a network of autonomous services that can run complex logic in a decentralized manner, interacting with on- and off-chain data autonomously and continuously. For other use cases check out [olas.network](https://olas.network/). 
+Since 'Olas' means 'waves' in Spanish, it is sometimes referred to as the 'ocean of services' 🌊.
+The project is co-created by [Valory](https://www.valory.xyz/). Valory aspires to enable communities, organizations and countries to co-own AI systems, beginning with decentralized autonomous agents.
+"""

tabs/howto_benchmark.py

@@ -0,0 +1,26 @@
+how_to_run = """\
+## Create a new Tool
+
+- Fork the repo [here](https://github.com/valory-xyz/mech).
+- Create your tool in mech/packages/{your-name-here}/f'{your-tool}.py'
+- Submit a PR
+
+## Test your Tool on the Benchmark On HF
+
+- Navigate to the "🔥 Run the Benchmark" page
+- Choose your tool and model
+- Enter your API key
+- Press "Run Benchmark"
+- Inspect the results
+
+## Test your Tool on the Benchmark locally
+
+- Fork the repo [here](https://github.com/valory-xyz/olas-predict-benchmark).
+- Follow the instructions in the readme.
+
+## Run a Prediction Agent that uses these Tools to place real bets
+
+- Fork the repo [here](https://github.com/valory-xyz/trader-quickstart).
+- Follow the instructions in the readme.
+
+"""

tabs/run_benchmark.py

@@ -0,0 +1,44 @@
+import os
+import importlib
+import sys
+# from pathlib import Path
+from benchmark.run_benchmark import run_benchmark
+# from dotenv import load_dotenv
+
+
+# file_path = Path(__file__).resolve()
+# parent_path = file_path.parent.parent
+# load_dotenv(parent_path / ".env")
+
+
+def run_benchmark_main(tool_name, model_name, openai_api_key, anthropic_api_key):
+    """Run the benchmark using the provided function and API key."""
+    # Empyt the results directory
+    os.system("rm -rf results/*")
+
+    # Set the benchmark parameters
+    kwargs = {}
+    kwargs["num_questions"] = 2
+    kwargs["tools"] = [tool_name]
+    if model_name:
+        kwargs["model"] = [model_name]
+    kwargs["api_keys"] = {}
+    if openai_api_key:
+        kwargs["api_keys"]["openai"] = openai_api_key
+    if anthropic_api_key:
+        kwargs["api_keys"]["anthropic"] = anthropic_api_key
+
+    kwargs["num_urls"] = 3
+    kwargs["num_words"] = 300
+    kwargs["provide_source_links"] = True
+
+    print(f"Running benchmark with the following parameters: {kwargs}")
+    
+    # Run the benchmark
+    try:
+        run_benchmark(kwargs=kwargs)
+        return "completed"
+    except Exception as e:
+        return f"Error running benchmark: {e}"
+
+