import json from io import TextIOWrapper TITLE = """

Agent-Studio Leaderboard

""" INTRODUCTION_TEXT = """ AgentStudio is an open toolkit covering the entire lifespan of building virtual agents that can interact with everything on digital worlds. Here, we open-source the beta of environment implementations, benchmark suite, data collection pipeline, and graphical interfaces to promote research towards generalist virtual agents of the future. ## Submissions You should submit a zip file containing the agent-studio output. **Do not change the file names**. The file name is used to identify the scores of each category. ### Real-world tasks The file structure should be as follows: ``` results.zip ├── filesystem.jsonl ├── gcalendar.jsonl ├── gdocs.jsonl ├── gmail.jsonl ├── vscode.jsonl ├── desktop_hard.jsonl ├── ... ``` ### GUI grounding tasks The file structure should be as follows: ``` results.zip ├── linux │ ├── browser │ │ ├── results.jsonl | ├── os │ │ ├── results.jsonl │ ├── ... ├── windows | ├── word │ │ ├── results.jsonl | ├── os │ │ ├── results.jsonl │ ├── ... ├── macos ``` """ def format_error(msg): return f"

{msg}

" def format_warning(msg): return f"

{msg}

" def format_log(msg): return f"

{msg}

" def model_hyperlink(link, model_name): return f'{model_name}' def read_jsonl(file: str | TextIOWrapper, start_idx: int = 0, end_idx: int | None = None) -> list: """Reads lines from a .jsonl file between start_idx and end_idx. Args: file (str | TextIOWrapper): Path to the .jsonl file or an open file object start_idx (int, optional): The starting index of lines to read end_idx (int | None, optional): The ending index of lines to read Returns: list[dict]: A list of dictionaries, each dictionary is a line from the .jsonl file """ if end_idx is not None and start_idx > end_idx: raise ValueError("start_idx must be less or equal to end_idx") data = [] if isinstance(file, str): with open(file, "r") as file: for i, line in enumerate(file): if end_idx is not None and i >= end_idx: break if i >= start_idx: data.append(json.loads(line)) else: for i, line in enumerate(file): if end_idx is not None and i >= end_idx: break if i >= start_idx: data.append(json.loads(line)) return data def add_jsonl(data: list, file: str, mode="a"): """Adds a list of dictionaries to a .jsonl file. Args: data (list[dict]): A list of json objects to add to the file file (str): Path to the .jsonl file """ with open(file, mode) as file: for item in data: json_str = json.dumps(item) file.write(json_str + "\n")