Move from markdown table to csv table as easier to maintain for larger tables

.vscode/extensions.json

@@ -1,5 +1,5 @@
 {
     "recommendations": [
-        "takumii.markdowntable"
+        "janisdd.vscode-edit-csv"
     ]
 }

README.md

@@ -1,18 +1,13 @@
 # llm-leaderboard
-A joint community effort to create one central leaderboard for LLMs
+A joint community effort to create one central leaderboard for LLMs. Contributions and corrections welcome!
 
-Visit the interactive leaderboard at TODO.
+## Leaderboard
 
-### Leaderboard
+Visit the interactive leaderboard at https://llm-leaderboard.streamlit.app/.
+
+## How to contribute
+
+You can contribute by:
+- adding a new model as a new row
+- adding a new benchmark as a new column
 
-| Model Name                                                                                                            | [Chatbot Arena Elo (llmsys)](https://lmsys.org/blog/2023-05-03-arena/) |
-| --------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- |
-| [alpaca-13b](https://crfm.stanford.edu/2023/03/13/alpaca.html)                                                        | 1008                                                                   |
-| [chatglm-6b](https://chatglm.cn/blog)                                                                                 | 985                                                                    |
-| [dolly-v2-12b](https://www.databricks.com/blog/2023/04/12/dolly-first-open-commercially-viable-instruction-tuned-llm) | 944                                                                    |
-| [fastchat-t5-3b](https://huggingface.co/lmsys/fastchat-t5-3b-v1.0)                                                    | 951                                                                    |
-| [koala-13b](https://bair.berkeley.edu/blog/2023/04/03/koala/)                                                         | 1082                                                                   |
-| [llama-13b](https://ai.facebook.com/blog/large-language-model-llama-meta-ai/)                                         | 932                                                                    |
-| [stablelm-tuned-alpha-7b](https://github.com/stability-AI/stableLM)                                                   | 858                                                                    |
-| [vicuna-13b](https://lmsys.org/blog/2023-03-30-vicuna/)                                                               | 1169                                                                   |
-| [oasst-pythia-12b](https://open-assistant.io/)                                                                        | 1065                                                                   |

requirements-dev.txt

@@ -1,3 +1,4 @@
 black
 flake
+isort
 mypy

streamlit_app.py

@@ -1,130 +1,24 @@
-import pandas as pd
-import streamlit as st
 import io
+
+import pandas as pd
 import requests
-import re
+import streamlit as st
 
 REPO_URL = "https://github.com/LudwigStumpp/llm-leaderboard"
 
 
-def grab_readme_file_from_repo(repo_url: str) -> str:
-    """Grabs the README.md file from a GitHub repository.
+def grab_file_from_repo(repo_url: str, filename: str) -> str:
+    """Grabs a file from a GitHub repository.
 
     Args:
         repo_url (str): URL of the GitHub repository.
+        filename (str): Name of the file to grab.
 
     Returns:
-        str: Content of the README.md file.
-    """
-    readme_url = repo_url.replace("github.com", "raw.githubusercontent.com") + "/main/README.md"
-    readme = requests.get(readme_url).text
-    return readme
-
-
-def modify_from_markdown_links_to_html_links(text: str) -> str:
-    """Modifies a markdown text to replace all markdown links with HTML links.
-
-    Example: [DISPLAY](LINK) to <a href=LINK, target="_blank">DISPLAY</a>
-
-    First find all markdown links with regex.
-    Then replace them with: <a href=$2, target="_blank">$1</a>
-
-    Args:
-        text (str): Markdown text containing markdown links
-
-    Returns:
-        str: Markdown text with HTML links.
-    """
-
-    # find all markdown links
-    markdown_links = re.findall(r"\[([^\]]+)\]\(([^)]+)\)", text)
-
-    # replace them with HTML links
-    for display, link in markdown_links:
-        text = text.replace(f"[{display}]({link})", f'<a href="{link}" target="_blank">{display}</a>')
-
-    return text
-
-
-def remove_markdown_links(text: str) -> str:
-    """Modifies a markdown text to remove all markdown links.
-
-    Example: [DISPLAY](LINK) to DISPLAY
-
-    First find all markdown links with regex.
-    Then replace them with: $1
-
-    Args:
-        text (str): Markdown text containing markdown links
-
-    Returns:
-        str: Markdown text without markdown links.
+        str: Content of the file.
     """
-
-    # find all markdown links
-    markdown_links = re.findall(r"\[([^\]]+)\]\(([^)]+)\)", text)
-
-    # remove link keep display text
-    for display, link in markdown_links:
-        text = text.replace(f"[{display}]({link})", display)
-
-    return text
-
-
-def extract_table_and_format_from_markdown_text(markdown_table: str) -> pd.DataFrame:
-    """Extracts a table from a markdown text and formats it as a pandas DataFrame.
-
-    Args:
-        text (str): Markdown text containing a table.
-
-    Returns:
-        pd.DataFrame: Table as pandas DataFrame.
-    """
-    df = (
-        pd.read_table(io.StringIO(markdown_table), sep="|", header=0, index_col=1)
-        .dropna(axis=1, how="all")  # drop empty columns
-        .iloc[1:]  # drop first row which is the "----" separator of the original markdown table
-    )
-
-    # change all column datatypes to numeric
-    for col in df.columns:
-        df[col] = pd.to_numeric(df[col], errors="ignore")
-
-    # remove whitespace from column names and index
-    df.columns = df.columns.str.strip()
-    df.index = df.index.str.strip()
-
-    return df
-
-
-def extract_markdown_table_from_multiline(multiline: str, table_headline: str) -> str:
-    """Extracts the markdown table from a multiline string.
-
-    Args:
-        multiline (str): content of README.md file.
-        table_headline (str): Headline of the table in the README.md file.
-
-    Returns:
-        str: Markdown table.
-
-    Raises:
-        ValueError: If the table could not be found.
-    """
-    # extract everything between the table headline and the next headline
-    table = []
-    start = False
-    for line in multiline.split("\n"):
-        if line.startswith(table_headline):
-            start = True
-        elif line.startswith("###"):
-            start = False
-        elif start:
-            table.append(line + "\n")
-
-    if len(table) == 0:
-        raise ValueError(f"Could not find table with headline '{table_headline}'")
-
-    return "".join(table)
+    url = repo_url.replace("github.com", "raw.githubusercontent.com") + f"/main/{filename}"
+    return requests.get(url).text
 
 
 def setup_basic():
@@ -145,10 +39,8 @@ def setup_basic():
 
 
 def setup_table():
-    readme = grab_readme_file_from_repo(REPO_URL)
-    markdown_table = extract_markdown_table_from_multiline(readme, table_headline="### Leaderboard")
-    markdown_table = remove_markdown_links(markdown_table)
-    df = extract_table_and_format_from_markdown_text(markdown_table)
+    csv_table = grab_file_from_repo(REPO_URL, "leaderboard.csv")
+    df = pd.read_csv(io.StringIO(csv_table), index_col=0)
     st.dataframe(df)