guipenedo HF staff commited on
Commit
28d348e
1 Parent(s): b25c99e

initial upload

Browse files
Files changed (2) hide show
  1. .gitignore +163 -0
  2. app.py +83 -0
.gitignore ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py,cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # poetry
98
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102
+ #poetry.lock
103
+
104
+ # pdm
105
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106
+ #pdm.lock
107
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108
+ # in version control.
109
+ # https://pdm.fming.dev/#use-with-ide
110
+ .pdm.toml
111
+
112
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113
+ __pypackages__/
114
+
115
+ # Celery stuff
116
+ celerybeat-schedule
117
+ celerybeat.pid
118
+
119
+ # SageMath parsed files
120
+ *.sage.py
121
+
122
+ # Environments
123
+ .env
124
+ .venv
125
+ env/
126
+ venv/
127
+ ENV/
128
+ env.bak/
129
+ venv.bak/
130
+
131
+ # Spyder project settings
132
+ .spyderproject
133
+ .spyproject
134
+
135
+ # Rope project settings
136
+ .ropeproject
137
+
138
+ # mkdocs documentation
139
+ /site
140
+
141
+ # mypy
142
+ .mypy_cache/
143
+ .dmypy.json
144
+ dmypy.json
145
+
146
+ # Pyre type checker
147
+ .pyre/
148
+
149
+ # pytype static type analyzer
150
+ .pytype/
151
+
152
+ # Cython debug symbols
153
+ cython_debug/
154
+
155
+ # PyCharm
156
+ .idea/
157
+
158
+ # ruff code style
159
+ .ruff_cache/
160
+
161
+ .vscode/
162
+
163
+ playground/
app.py ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ast
2
+ import json
3
+ import urllib
4
+
5
+ import gradio as gr
6
+
7
+ import pandas as pd
8
+ from datasets import load_dataset
9
+ from gradio_leaderboard import Leaderboard, SelectColumns, SearchColumns, ColumnFilter
10
+
11
+ df = pd.read_json("https://raw.githubusercontent.com/huggingface/lighteval/main/src/lighteval/tasks/tasks_table.jsonl", lines=True).explode("suite").reset_index(drop=True)
12
+
13
+ with urllib.request.urlopen("https://raw.githubusercontent.com/huggingface/lighteval/main/src/lighteval/tasks/tasks_prompt_formatting.py") as f:
14
+ tasks_prompt_functions_raw = f.read().decode('utf-8')
15
+ tree = ast.parse(tasks_prompt_functions_raw)
16
+ tasks_prompt_functions = {}
17
+
18
+ for node in ast.walk(tree):
19
+ if isinstance(node, ast.FunctionDef):
20
+ function_name = node.name
21
+ # Get the source code for the function node
22
+ function_code = ast.get_source_segment(tasks_prompt_functions_raw, node)
23
+ tasks_prompt_functions[function_name] = function_code
24
+
25
+ def load_task_metadata(task_id):
26
+ task_row = df.iloc[task_id]
27
+ return (task_row.to_dict(), f"""Examples from the HF repository ([{task_row['hf_repo']}](https://huggingface.co/datasets/{task_row['hf_repo']}))""",
28
+ tasks_prompt_functions.get(task_row["prompt_function"]), "unknown")
29
+
30
+ def load_task_examples(task_id):
31
+ task_row = df.iloc[task_id]
32
+ dataset = load_dataset(task_row["hf_repo"], task_row["hf_subset"], split="+".join(task_row["evaluation_splits"]),
33
+ trust_remote_code=task_row["trust_dataset"], streaming=True)
34
+
35
+ sample_data = next(dataset.iter(20))
36
+ # dictionary of lists to list of dictionaries
37
+ return pd.DataFrame(dict(zip(sample_data, t if not isinstance(t, dict) and not isinstance(t, list) else json.dumps(t)))
38
+ for t in zip(*sample_data.values()))
39
+
40
+
41
+
42
+ with gr.Blocks() as demo:
43
+ gr.Markdown("""
44
+ # LightEval Tasks Explorer
45
+ """)
46
+ with gr.Tabs() as tabs:
47
+ with gr.TabItem("🗃️ Tasks List"):
48
+ Leaderboard(
49
+ value=df,
50
+ select_columns=SelectColumns(default_selection=["name", "suite", "prompt_function", "hf_repo", "hf_subset", "evaluation_splits", "metric"],
51
+ cant_deselect=["name", "suite"],
52
+ label="Columns to display"),
53
+ search_columns=SearchColumns(primary_column="name", secondary_columns=["suite", "prompt_function", "hf_repo", "metric"],
54
+ placeholder="Search for a task by name, suite, prompt_function, hf_repo or "
55
+ "metric. To search by suite, for example, type 'suite:<query>'. Separate queries by \";\"",
56
+ label="Search"),
57
+ filter_columns=[
58
+ ColumnFilter("suite", type="dropdown", label="Select suite"),
59
+ # ColumnFilter("prompt_function", type="dropdown", label="Select prompt_function"),
60
+ # ColumnFilter("metric", type="dropdown", label="Select metric")
61
+ ],
62
+ wrap=True
63
+ )
64
+ with gr.TabItem("🔎 Task Inspector"):
65
+ task_inspector_selector = gr.Dropdown(
66
+ choices=sorted(zip((df['suite'] + '|' + df['name']).tolist(), range(len(df)))),
67
+ label="Task",
68
+ info="Select a task"
69
+ )
70
+ with gr.Row():
71
+ with gr.Column():
72
+ task_metadata = gr.Json(label="Task definition")
73
+ with gr.Column():
74
+ task_prompt_function = gr.Code(label="Task prompt function", language="python", interactive=False)
75
+ task_dataset_header = gr.Markdown("Examples from the HF repository")
76
+ task_dataset = gr.Dataframe(wrap=True)
77
+
78
+ gr.on(triggers=[task_inspector_selector.change], inputs=[task_inspector_selector], outputs=[task_metadata, task_dataset_header, task_prompt_function], fn=load_task_metadata)
79
+ gr.on(triggers=[task_inspector_selector.change], inputs=[task_inspector_selector], outputs=[task_dataset], fn=load_task_examples)
80
+
81
+
82
+ if __name__ == "__main__":
83
+ demo.launch()