Add functions to submit models

app.py

@@ -1,37 +1,38 @@
 import gradio as gr
 
-from leaderboard.dataset import load_or_initialize_leaderboard
+from leaderboard.dataset import get_leaderboard_df
 from leaderboard.submission import submit_model
 
 
-# リーダーボード表示
 def display_leaderboard():
-    dataset = load_or_initialize_leaderboard()
-    return dataset.to_pandas()
+    df = get_leaderboard_df()
+    return df
 
 
-# Gradio のコンポーネント
-leaderboard_component = gr.DataFrame(
-    display_leaderboard, headers=["Model Name", "Score", "Rank"], interactive=False, label="Leaderboard"
-)
-
-submit_form = gr.Interface(
-    submit_model,
-    inputs=[gr.Textbox(label="Model Name"), gr.File(label="Model File")],
-    outputs=gr.DataFrame(headers=["Model Name", "Score", "Rank"], interactive=False),
-)
-
-# Gradio アプリケーション
-app = gr.Blocks()
-
-with app:
-    gr.Markdown("# human_methylation_bench_ver1")
+with gr.Blocks() as app:
+    gr.Markdown("# human_methylation_bench_ver1 Leaderboard")
 
     with gr.Tab("Leaderboard"):
-        leaderboard_component.render()
+        leaderboard_df = gr.DataFrame(
+            value=display_leaderboard(),
+            headers=["Model Name", "Score (relative_error_loss)", "Rank"],
+            interactive=False,
+            label="Leaderboard",
+        )
 
     with gr.Tab("Submit Model"):
-        submit_form.render()
+        model_name_input = gr.Textbox(label="Model Name", placeholder="e.g. My Great Model")
+        model_url_input = gr.Textbox(
+            label="Hugging Face Model Joblib URL",
+            placeholder="e.g. https://huggingface.co/username/model/resolve/main/model.joblib",
+        )
+        submit_button = gr.Button("Submit")
+
+        submission_output = gr.DataFrame(
+            headers=["Model Name", "Score (relative_error_loss)", "Rank"], interactive=False, label="Updated Leaderboard"
+        )
+
+        submit_button.click(submit_model, inputs=[model_name_input, model_url_input], outputs=submission_output)
 
 if __name__ == "__main__":
     app.launch()

leaderboard/dataset.py

@@ -1,25 +1,27 @@
+import os
+
+import pandas as pd
 from datasets import Dataset, load_dataset
+from dotenv import load_dotenv
+from huggingface_hub import login
 
-DATASET_PATH = "leaderboard_dataset"
+load_dotenv()
 
-# 初期データ
-INITIAL_DATA = {
-    "Model Name": ["Baseline Model"],
-    "Score": [0.8],
-    "Rank": [1],
-}
+HF_TOKEN = os.environ.get("HF_TOKEN", None)
+RESULT_DATASET_ID = os.environ.get("RESULT_DATASET_ID", None)
+if HF_TOKEN:
+    login(token=HF_TOKEN)
 
 
-# データセットを初期化またはロード
-def load_or_initialize_leaderboard():
-    try:
-        dataset = Dataset.load_from_disk(DATASET_PATH)
-    except FileNotFoundError:
-        dataset = Dataset.from_dict(INITIAL_DATA)
-        dataset.save_to_disk(DATASET_PATH)
-    return dataset
+def get_leaderboard_df() -> pd.DataFrame:
+    # リーダーボードデータセットをHugging Face Hubからロード
+    ds = load_dataset(RESULT_DATASET_ID, split="train")
+    df = ds.to_pandas()
+    return df
 
 
-# データセットを保存
-def save_leaderboard(dataset):
-    dataset.save_to_disk(DATASET_PATH)
+def save_leaderboard_df(df: pd.DataFrame):
+    # DataFrameをDataset化
+    ds = Dataset.from_pandas(df, preserve_index=False)
+    # push_to_hubで更新
+    ds.push_to_hub(RESULT_DATASET_ID, token=HF_TOKEN, commit_message="Update leaderboard")

leaderboard/evaluation.py

@@ -1,8 +1,63 @@
-# ダミーの評価関数
-def evaluate_model(model_path):
-    """
-    提出モデルを評価してスコアを返す関数。
-    本番ではモデルをロードしてテストデータに基づくスコアを計算する。
-    """
-    # TODO: 実際の評価ロジックを実装する
-    return 0.75  # 仮のスコア
+import os
+import re
+import tempfile
+from urllib.parse import urlparse
+
+import joblib
+import numpy as np
+from datasets import load_dataset
+from dotenv import load_dotenv
+from huggingface_hub import hf_hub_download
+
+load_dotenv()
+
+TEST_DATA_ID = os.environ.get("TEST_DATA_ID", None)
+
+
+def relative_error_loss(predicted_age, true_age):
+    true_age_safe = np.where(true_age == 0, 0.1, true_age)
+    relative_error = np.abs((true_age - predicted_age) / true_age_safe)
+    return np.mean(relative_error)
+
+
+def parse_model_url(model_url: str):
+    parsed = urlparse(model_url)
+    path_parts = parsed.path.strip("/").split("/")
+
+    if len(path_parts) < 5:
+        raise ValueError("Unexpected URL format. Make sure it's a Hub URL with /resolve/main/ or /blob/main/")
+
+    repo_id = "/".join(path_parts[:2])
+    revision = path_parts[3]
+    filename = path_parts[4]
+
+    if not filename.endswith(".joblib"):
+        raise ValueError("The file must be a .joblib file.")
+
+    return repo_id, revision, filename
+
+
+def evaluate_model(model_url: str) -> float:
+    if not model_url.startswith("https://huggingface.co/"):
+        raise ValueError("Invalid model URL. Must start with https://huggingface.co/")
+
+    repo_id, revision, filename = parse_model_url(model_url)
+
+    ds_test_meta = load_dataset(TEST_DATA_ID, "meta")
+    ds_test_main = load_dataset(TEST_DATA_ID, "main")
+
+    X_test = ds_test_main["test"].to_pandas().drop(columns=["SampleID"])
+    X_test = X_test.values.astype(np.float32)
+    y_test = np.array(ds_test_meta["test"]["Age"])
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        local_model_path = hf_hub_download(repo_id=repo_id, filename=filename, revision=revision, cache_dir=tmpdir)
+        try:
+            model = joblib.load(local_model_path)
+        except Exception as e:
+            raise ValueError(f"Failed to load the model. Please check the .joblib file. Error: {e}")
+
+        predicted_age = model.predict(X_test)
+        score = relative_error_loss(predicted_age, y_test)
+
+    return score

leaderboard/submission.py

@@ -1,32 +1,30 @@
-from datasets import Dataset
+import pandas as pd
 
-from .dataset import load_or_initialize_leaderboard, save_leaderboard
+from .dataset import get_leaderboard_df, save_leaderboard_df
 from .evaluation import evaluate_model
 
 
-def submit_model(model_name, model_file):
-    """
-    モデルの提出を処理する関数。
-    1. モデルを評価する。
-    2. リーダーボードにデータを追加。
-    3. ランクを計算して保存。
-    """
-    dataset = load_or_initialize_leaderboard()
+def submit_model(model_name: str, model_url: str):
+    if not model_name.strip():
+        raise ValueError("Model name cannot be empty.")
+    if not model_url.strip():
+        raise ValueError("Model URL cannot be empty.")
 
-    # モデル評価
-    score = evaluate_model(model_file.name)
+    # 現在のリーダーボードを取得
+    df = get_leaderboard_df()
 
-    # データに新しいモデルを追加
-    new_entry = {"Model Name": model_name, "Score": score}
-    dataset = dataset.add_item(new_entry)
+    # 新規評価
+    score = evaluate_model(model_url)
 
-    # ランク付け
-    df = dataset.to_pandas()
-    df = df.sort_values(by="Score", ascending=False).reset_index(drop=True)
+    # 新規行追加
+    new_entry = {"Model Name": model_name.strip(), "Score (relative_error_loss)": score}
+    df = pd.concat([df, pd.DataFrame([new_entry])], ignore_index=True)
+
+    # スコアが小さいほど良いと仮定し、昇順ソート＆Rank再計算
+    df = df.sort_values(by="Score (relative_error_loss)", ascending=True).reset_index(drop=True)
     df["Rank"] = range(1, len(df) + 1)
 
-    # データセットを更新・保存
-    updated_dataset = Dataset.from_pandas(df)
-    save_leaderboard(updated_dataset)
+    # データセットをHugging Face Hubに反映
+    save_leaderboard_df(df)
 
     return df

pyproject.toml

@@ -9,7 +9,12 @@ readme = "README.md"
 python = "^3.12"
 gradio = "^5.6.0"
 pandas = "^2.2.3"
-datasets = "^3.1.0"
+joblib = "^1.4.2"
+scikit-learn = "^1.6.0"
+datasets = "^3.2.0"
+
+[tool.poetry.group.dev.dependencies]
+python-dotenv = "^1.0.1"
 
 [build-system]
 requires = ["poetry-core"]

requirements.txt

@@ -1,16 +1,71 @@
-APScheduler
-black
-datasets
-gradio
-gradio[oauth]
-gradio_leaderboard==0.0.9
-gradio_client
-huggingface-hub>=0.18.0
-matplotlib
-numpy
-pandas
-python-dateutil
-tqdm
-transformers
-tokenizers>=0.15.0
-sentencepiece
+aiofiles==23.2.1 ; python_version >= "3.12" and python_version < "4.0"
+aiohappyeyeballs==2.4.4 ; python_version >= "3.12" and python_version < "4.0"
+aiohttp==3.11.10 ; python_version >= "3.12" and python_version < "4.0"
+aiosignal==1.3.1 ; python_version >= "3.12" and python_version < "4.0"
+annotated-types==0.7.0 ; python_version >= "3.12" and python_version < "4.0"
+anyio==4.6.2.post1 ; python_version >= "3.12" and python_version < "4.0"
+attrs==24.2.0 ; python_version >= "3.12" and python_version < "4.0"
+audioop-lts==0.2.1 ; python_version >= "3.13" and python_version < "4.0"
+certifi==2024.8.30 ; python_version >= "3.12" and python_version < "4.0"
+charset-normalizer==3.4.0 ; python_version >= "3.12" and python_version < "4.0"
+click==8.1.7 ; python_version >= "3.12" and python_version < "4.0" and sys_platform != "emscripten"
+colorama==0.4.6 ; python_version >= "3.12" and python_version < "4.0" and platform_system == "Windows"
+datasets==3.2.0 ; python_version >= "3.12" and python_version < "4.0"
+dill==0.3.8 ; python_version >= "3.12" and python_version < "4.0"
+fastapi==0.115.5 ; python_version >= "3.12" and python_version < "4.0"
+ffmpy==0.4.0 ; python_version >= "3.12" and python_version < "4.0"
+filelock==3.16.1 ; python_version >= "3.12" and python_version < "4.0"
+frozenlist==1.5.0 ; python_version >= "3.12" and python_version < "4.0"
+fsspec==2024.9.0 ; python_version >= "3.12" and python_version < "4.0"
+fsspec[http]==2024.9.0 ; python_version >= "3.12" and python_version < "4.0"
+gradio-client==1.4.3 ; python_version >= "3.12" and python_version < "4.0"
+gradio==5.6.0 ; python_version >= "3.12" and python_version < "4.0"
+h11==0.14.0 ; python_version >= "3.12" and python_version < "4.0"
+httpcore==1.0.7 ; python_version >= "3.12" and python_version < "4.0"
+httpx==0.27.2 ; python_version >= "3.12" and python_version < "4.0"
+huggingface-hub==0.26.2 ; python_version >= "3.12" and python_version < "4.0"
+idna==3.10 ; python_version >= "3.12" and python_version < "4.0"
+jinja2==3.1.4 ; python_version >= "3.12" and python_version < "4.0"
+joblib==1.4.2 ; python_version >= "3.12" and python_version < "4.0"
+markdown-it-py==3.0.0 ; python_version >= "3.12" and python_version < "4.0" and sys_platform != "emscripten"
+markupsafe==2.1.5 ; python_version >= "3.12" and python_version < "4.0"
+mdurl==0.1.2 ; python_version >= "3.12" and python_version < "4.0" and sys_platform != "emscripten"
+multidict==6.1.0 ; python_version >= "3.12" and python_version < "4.0"
+multiprocess==0.70.16 ; python_version >= "3.12" and python_version < "4.0"
+numpy==2.1.3 ; python_version >= "3.12" and python_version < "4.0"
+orjson==3.10.12 ; python_version >= "3.12" and python_version < "4.0"
+packaging==24.2 ; python_version >= "3.12" and python_version < "4.0"
+pandas==2.2.3 ; python_version >= "3.12" and python_version < "4.0"
+pillow==11.0.0 ; python_version >= "3.12" and python_version < "4.0"
+propcache==0.2.1 ; python_version >= "3.12" and python_version < "4.0"
+pyarrow==18.1.0 ; python_version >= "3.12" and python_version < "4.0"
+pydantic-core==2.27.1 ; python_version >= "3.12" and python_version < "4.0"
+pydantic==2.10.1 ; python_version >= "3.12" and python_version < "4.0"
+pydub==0.25.1 ; python_version >= "3.12" and python_version < "4.0"
+pygments==2.18.0 ; python_version >= "3.12" and python_version < "4.0" and sys_platform != "emscripten"
+python-dateutil==2.9.0.post0 ; python_version >= "3.12" and python_version < "4.0"
+python-multipart==0.0.12 ; python_version >= "3.12" and python_version < "4.0"
+pytz==2024.2 ; python_version >= "3.12" and python_version < "4.0"
+pyyaml==6.0.2 ; python_version >= "3.12" and python_version < "4.0"
+requests==2.32.3 ; python_version >= "3.12" and python_version < "4.0"
+rich==13.9.4 ; python_version >= "3.12" and python_version < "4.0" and sys_platform != "emscripten"
+ruff==0.8.0 ; python_version >= "3.12" and python_version < "4.0" and sys_platform != "emscripten"
+safehttpx==0.1.1 ; python_version >= "3.12" and python_version < "4.0"
+scikit-learn==1.6.0 ; python_version >= "3.12" and python_version < "4.0"
+scipy==1.14.1 ; python_version >= "3.12" and python_version < "4.0"
+semantic-version==2.10.0 ; python_version >= "3.12" and python_version < "4.0"
+shellingham==1.5.4 ; python_version >= "3.12" and python_version < "4.0" and sys_platform != "emscripten"
+six==1.16.0 ; python_version >= "3.12" and python_version < "4.0"
+sniffio==1.3.1 ; python_version >= "3.12" and python_version < "4.0"
+starlette==0.41.3 ; python_version >= "3.12" and python_version < "4.0"
+threadpoolctl==3.5.0 ; python_version >= "3.12" and python_version < "4.0"
+tomlkit==0.12.0 ; python_version >= "3.12" and python_version < "4.0"
+tqdm==4.67.1 ; python_version >= "3.12" and python_version < "4.0"
+typer==0.13.1 ; python_version >= "3.12" and python_version < "4.0" and sys_platform != "emscripten"
+typing-extensions==4.12.2 ; python_version >= "3.12" and python_version < "4.0"
+tzdata==2024.2 ; python_version >= "3.12" and python_version < "4.0"
+urllib3==2.2.3 ; python_version >= "3.12" and python_version < "4.0"
+uvicorn==0.32.1 ; python_version >= "3.12" and python_version < "4.0" and sys_platform != "emscripten"
+websockets==12.0 ; python_version >= "3.12" and python_version < "4.0"
+xxhash==3.5.0 ; python_version >= "3.12" and python_version < "4.0"
+yarl==1.18.3 ; python_version >= "3.12" and python_version < "4.0"