|
import os |
|
import sys |
|
sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "./leaderboard/evaluation"))) |
|
sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "./leaderboard"))) |
|
os.chdir(os.path.dirname(os.path.abspath(__file__))) |
|
os.environ['CURL_CA_BUNDLE'] = '' |
|
import json |
|
from datetime import datetime |
|
from email.utils import parseaddr |
|
|
|
import gradio as gr |
|
import pandas as pd |
|
import numpy as np |
|
|
|
from datasets import load_dataset |
|
from apscheduler.schedulers.background import BackgroundScheduler |
|
from huggingface_hub import HfApi |
|
|
|
|
|
|
|
from content import format_error, format_warning, format_log, TITLE, INTRODUCTION_TEXT, CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, model_hyperlink |
|
from eval import eval_score |
|
|
|
TOKEN = os.environ.get("TOKEN", None) |
|
OWNER="osunlp" |
|
DATA_DATASET = f"{OWNER}/TravelPlanner" |
|
EVAL_DATASET = f"{OWNER}/TravelPlannerEval" |
|
RESULTS_DATASET = f"{OWNER}/TravelPlannerPublicResults" |
|
|
|
api = HfApi() |
|
|
|
|
|
|
|
os.makedirs("scored", exist_ok=True) |
|
|
|
|
|
eval_results = load_dataset(RESULTS_DATASET, 'scores', token=TOKEN, download_mode="force_redownload", ignore_verifications=True) |
|
def get_dataframe_from_results(eval_results, split, mode): |
|
local_df = eval_results[f'{split}_{mode}'] |
|
local_df = local_df.remove_columns(["Mail"]) |
|
df = pd.DataFrame(local_df) |
|
df['Organization'].mask(df['Organization']=='TravelBench Team','TravelPlanner Team',inplace=True) |
|
df = df.sort_values(by=["Final Pass Rate"], ascending=False) |
|
numeric_cols = [c for c in local_df.column_names if "Rate" in c] |
|
df[numeric_cols] = df[numeric_cols].multiply(100).round(decimals=2) |
|
return df |
|
|
|
|
|
eval_dataframe_val_twostage = get_dataframe_from_results(eval_results=eval_results, split="validation", mode='twostage') |
|
eval_dataframe_val_soleplanning = get_dataframe_from_results(eval_results=eval_results, split="validation", mode='soleplanning') |
|
eval_dataframe_test_twostage = get_dataframe_from_results(eval_results=eval_results, split="test",mode='twostage') |
|
eval_dataframe_test_soleplanning = get_dataframe_from_results(eval_results=eval_results, split="test",mode='soleplanning') |
|
|
|
|
|
|
|
|
|
|
|
|
|
def load_line_json_data(filename): |
|
data = [] |
|
with open(filename, 'r', encoding='utf-8') as f: |
|
for line in f.read().strip().split('\n'): |
|
unit = json.loads(line) |
|
data.append(unit) |
|
return data |
|
|
|
|
|
def add_new_eval( |
|
val_or_test: str, |
|
eval_mode: str, |
|
path_to_file: str, |
|
): |
|
print("Adding new eval") |
|
|
|
if path_to_file is None: |
|
return format_warning("Please attach a file.") |
|
|
|
|
|
|
|
file_path = path_to_file.name |
|
result, detail_json = eval_score(val_or_test,file_path=file_path,TOKEN=TOKEN) |
|
print(detail_json) |
|
print(type(detail_json)) |
|
outputPath=os.path.join('.',datetime.now().strftime('%Y%m%d%H%M%S') + '.json') |
|
with open(outputPath,'w') as w: |
|
json.dump(detail_json,w) |
|
|
|
return format_log(f"{result}"), gr.File(label=f"Download the detailed constraint pass rate reports", value=outputPath, visible=True) |
|
|
|
|
|
def refresh(): |
|
eval_results = load_dataset(RESULTS_DATASET, 'scores', token=TOKEN, download_mode="force_redownload", ignore_verifications=True) |
|
eval_dataframe_val_twostage = get_dataframe_from_results(eval_results=eval_results, split="validation", mode='twostage') |
|
eval_dataframe_val_soleplanning = get_dataframe_from_results(eval_results=eval_results, split="validation", mode='soleplanning') |
|
eval_dataframe_test_twostage = get_dataframe_from_results(eval_results=eval_results, split="test",mode='twostage') |
|
eval_dataframe_test_soleplanning = get_dataframe_from_results(eval_results=eval_results, split="test",mode='soleplanning') |
|
return eval_dataframe_val_twostage, eval_dataframe_val_soleplanning, eval_dataframe_test_twostage, eval_dataframe_test_soleplanning |
|
|
|
|
|
|
|
demo = gr.Blocks() |
|
with demo: |
|
gr.HTML(TITLE) |
|
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") |
|
|
|
with gr.Tab("Results: Validation | Two-Stage "): |
|
leaderboard_table_val_twostage = gr.components.Dataframe( |
|
value=eval_dataframe_val_twostage, interactive=False, |
|
) |
|
with gr.Tab("Results: Validation | Sole-Planning"): |
|
leaderboard_table_val_soleplanning = gr.components.Dataframe( |
|
value=eval_dataframe_val_soleplanning, interactive=False, |
|
) |
|
|
|
with gr.Tab("Results: Test | Two-Stage "): |
|
leaderboard_table_test_twostage = gr.components.Dataframe( |
|
value=eval_dataframe_test_twostage, interactive=False, |
|
) |
|
with gr.Tab("Results: Test | Sole-Planning"): |
|
leaderboard_table_test_soleplanning = gr.components.Dataframe( |
|
value=eval_dataframe_test_soleplanning, interactive=False, |
|
) |
|
|
|
refresh_button = gr.Button("Refresh") |
|
refresh_button.click( |
|
refresh, |
|
inputs=[], |
|
outputs=[ |
|
leaderboard_table_val_twostage, |
|
leaderboard_table_val_soleplanning, |
|
leaderboard_table_test_twostage, |
|
leaderboard_table_test_soleplanning, |
|
], |
|
) |
|
with gr.Accordion("Submit a new file for evaluation"): |
|
with gr.Row(): |
|
with gr.Column(): |
|
level_of_test = gr.Radio(["validation", "test"], value="validation", label="Split") |
|
eval_mode = gr.Radio(["two-stage", "sole-planning"], value="two-stage", label="Eval Mode") |
|
file_input = gr.File(label="Upload file") |
|
file_output = gr.File(label="Download the detailed constraint pass rate reports", visible=False) |
|
|
|
submit_button = gr.Button("Submit Eval") |
|
submission_result = gr.Markdown() |
|
submit_button.click( |
|
add_new_eval, |
|
[ |
|
level_of_test, |
|
eval_mode, |
|
file_input, |
|
], |
|
[submission_result, file_output] |
|
) |
|
|
|
demo.launch(debug=True) |
|
|
|
|
|
|
|
|
