Spaces:
Runtime error
Runtime error
| import json | |
| from io import TextIOWrapper | |
| TITLE = """<h1 align="center" id="space-title">Agent-Studio Leaderboard</h1>""" | |
| INTRODUCTION_TEXT = """ | |
| AgentStudio is an open toolkit covering the entire lifespan of | |
| building virtual agents that can interact with everything on digital worlds. Here, we open-source the beta of environment implementations, benchmark suite, data collection pipeline, and graphical interfaces to promote research towards generalist virtual agents of the future. | |
| ## Submissions | |
| You should submit a zip file containing the agent-studio output. | |
| **Do not change the file names**. The file name is used to identify the scores of each category. | |
| ### Real-world tasks | |
| The file structure should be as follows: | |
| ``` | |
| results.zip | |
| ├── filesystem.jsonl | |
| ├── gcalendar.jsonl | |
| ├── gdocs.jsonl | |
| ├── gmail.jsonl | |
| ├── vscode.jsonl | |
| ├── desktop_hard.jsonl | |
| ├── ... | |
| ``` | |
| ### GUI grounding tasks | |
| The file structure should be as follows: | |
| ``` | |
| results.zip | |
| ├── linux | |
| │ ├── browser | |
| │ │ ├── results.jsonl | |
| | ├── os | |
| │ │ ├── results.jsonl | |
| │ ├── ... | |
| ├── windows | |
| | ├── word | |
| │ │ ├── results.jsonl | |
| | ├── os | |
| │ │ ├── results.jsonl | |
| │ ├── ... | |
| ├── macos | |
| ``` | |
| """ | |
| def format_error(msg): | |
| return f"<p style='color: red; font-size: 20px; text-align: center;'>{msg}</p>" | |
| def format_warning(msg): | |
| return f"<p style='color: orange; font-size: 20px; text-align: center;'>{msg}</p>" | |
| def format_log(msg): | |
| return f"<p style='color: green; font-size: 20px; text-align: center;'>{msg}</p>" | |
| def model_hyperlink(link, model_name): | |
| return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>' | |
| def read_jsonl(file: str | TextIOWrapper, start_idx: int = 0, end_idx: int | None = None) -> list: | |
| """Reads lines from a .jsonl file between start_idx and end_idx. | |
| Args: | |
| file (str | TextIOWrapper): Path to the .jsonl file or an open file object | |
| start_idx (int, optional): The starting index of lines to read | |
| end_idx (int | None, optional): The ending index of lines to read | |
| Returns: | |
| list[dict]: A list of dictionaries, each dictionary is a line from | |
| the .jsonl file | |
| """ | |
| if end_idx is not None and start_idx > end_idx: | |
| raise ValueError("start_idx must be less or equal to end_idx") | |
| data = [] | |
| if isinstance(file, str): | |
| with open(file, "r") as file: | |
| for i, line in enumerate(file): | |
| if end_idx is not None and i >= end_idx: | |
| break | |
| if i >= start_idx: | |
| data.append(json.loads(line)) | |
| else: | |
| for i, line in enumerate(file): | |
| if end_idx is not None and i >= end_idx: | |
| break | |
| if i >= start_idx: | |
| data.append(json.loads(line)) | |
| return data | |
| def add_jsonl(data: list, file: str, mode="a"): | |
| """Adds a list of dictionaries to a .jsonl file. | |
| Args: | |
| data (list[dict]): A list of json objects to add to the file | |
| file (str): Path to the .jsonl file | |
| """ | |
| with open(file, mode) as file: | |
| for item in data: | |
| json_str = json.dumps(item) | |
| file.write(json_str + "\n") | |