lewtun commited on
Commit
212696e
1 Parent(s): 3fc6417
Files changed (4) hide show
  1. .gitignore +141 -0
  2. Makefile +75 -0
  3. app.py +52 -0
  4. requirements.txt +3 -0
.gitignore ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py,cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow
98
+ __pypackages__/
99
+
100
+ # Celery stuff
101
+ celerybeat-schedule
102
+ celerybeat.pid
103
+
104
+ # SageMath parsed files
105
+ *.sage.py
106
+
107
+ # Environments
108
+ .env
109
+ .venv
110
+ env/
111
+ venv/
112
+ ENV/
113
+ env.bak/
114
+ venv.bak/
115
+
116
+ # Spyder project settings
117
+ .spyderproject
118
+ .spyproject
119
+
120
+ # Rope project settings
121
+ .ropeproject
122
+
123
+ # mkdocs documentation
124
+ /site
125
+
126
+ # mypy
127
+ .mypy_cache/
128
+ .dmypy.json
129
+ dmypy.json
130
+
131
+ # Pyre type checker
132
+ .pyre/
133
+
134
+ # pytype static type analyzer
135
+ .pytype/
136
+
137
+ # Cython debug symbols
138
+ cython_debug/
139
+
140
+ # Secrets
141
+ .env
Makefile ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ .PHONY: style
2
+
3
+ #################################################################################
4
+ # GLOBALS #
5
+ #################################################################################
6
+ DETECTED_OS := $(shell uname)
7
+
8
+ #################################################################################
9
+ # COMMANDS #
10
+ #################################################################################
11
+
12
+ ## Format code
13
+ style:
14
+ black --line-length 119 . && isort .
15
+
16
+ #################################################################################
17
+ # Self Documenting Commands #
18
+ #################################################################################
19
+
20
+ .DEFAULT_GOAL := help
21
+
22
+ # Inspired by <http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html>
23
+ # sed script explained:
24
+ # /^##/:
25
+ # * save line in hold space
26
+ # * purge line
27
+ # * Loop:
28
+ # * append newline + lin to hold space
29
+ # * go to next line
30
+ # * if line starts with doc comment, strip comment character off and loop
31
+ # * remove target prerequisites
32
+ # * append hold space (+ newline) to line
33
+ # * replace newline plus comments by `---`
34
+ # * print line
35
+ # Separate expressions are necessary because labels cannot be delimited by
36
+ # semicolon; see <http://stackoverflow.com/a/11799865/1968>
37
+ .PHONY: help
38
+ help:
39
+ @echo "$$(tput bold)Available rules:$$(tput sgr0)"
40
+ @echo
41
+ @sed -n -e "/^## / { \
42
+ h; \
43
+ s/.*//; \
44
+ :doc" \
45
+ -e "H; \
46
+ n; \
47
+ s/^## //; \
48
+ t doc" \
49
+ -e "s/:.*//; \
50
+ G; \
51
+ s/\\n## /---/; \
52
+ s/\\n/ /g; \
53
+ p; \
54
+ }" ${MAKEFILE_LIST} \
55
+ | LC_ALL='C' sort --ignore-case \
56
+ | awk -F '---' \
57
+ -v ncol=$$(tput cols) \
58
+ -v indent=19 \
59
+ -v col_on="$$(tput setaf 6)" \
60
+ -v col_off="$$(tput sgr0)" \
61
+ '{ \
62
+ printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
63
+ n = split($$2, words, " "); \
64
+ line_length = ncol - indent; \
65
+ for (i = 1; i <= n; i++) { \
66
+ line_length -= length(words[i]) + 1; \
67
+ if (line_length <= 0) { \
68
+ line_length = ncol - indent - length(words[i]) - 1; \
69
+ printf "\n%*s ", -indent, " "; \
70
+ } \
71
+ printf "%s ", words[i]; \
72
+ } \
73
+ printf "\n"; \
74
+ }' \
75
+ | more $(shell test $(shell uname) = Darwin && echo '--no-init --raw-control-chars')
app.py ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from pathlib import Path
3
+
4
+ import pandas as pd
5
+ import requests
6
+ import streamlit as st
7
+ from dotenv import load_dotenv
8
+
9
+ if Path(".env").is_file():
10
+ load_dotenv(".env")
11
+
12
+
13
+ def download_submissions():
14
+ auth_token = os.getenv("HF_HUB_TOKEN")
15
+ header = {"Authorization": "Bearer " + auth_token}
16
+ response = requests.get("http://huggingface.co/api/datasets", headers=header)
17
+ all_datasets = response.json()
18
+
19
+ submissions = []
20
+
21
+ for dataset in all_datasets:
22
+ is_benchmark = any([t for t in dataset["tags"] if t.split(":")[1] == "neelalex/raft-test"])
23
+ if is_benchmark:
24
+ submissions.append(dataset)
25
+
26
+ for submission in submissions:
27
+ submission_id = submission["id"]
28
+ response = requests.get(
29
+ f"http://huggingface.co/api/datasets/{submission_id}?full=true",
30
+ headers=header,
31
+ )
32
+ data = response.json()
33
+ submission_data = {"Team Name": [data["card_data"]["submission_dataset"].split("/")[0]]}
34
+ for task in data["card_data"]["model-index"]:
35
+ for task_name, splits in task.items():
36
+ for split in splits:
37
+ if split["split"] == "test":
38
+ for metric in split["metrics"]:
39
+ if metric["name"] == "f1":
40
+ submission_data[task_name] = [metric["value"]]
41
+
42
+ return pd.DataFrame(submission_data)
43
+
44
+
45
+ ###########
46
+ ### APP ###
47
+ ###########
48
+ st.set_page_config(layout="wide")
49
+ st.title("RAFT Leaderboard")
50
+ df = download_submissions()
51
+ # hack to remove index column from https://github.com/streamlit/streamlit/issues/641
52
+ st.table(df.assign(hack="").set_index("hack"))
requirements.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
1
+ streamlit
2
+ pandas
3
+ python-dotenv