nouamanetazi HF staff commited on
Commit
dc1b8a3
0 Parent(s):

initial commit

Browse files
.gitignore ADDED
@@ -0,0 +1,145 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py,cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # poetry
98
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102
+ #poetry.lock
103
+
104
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow
105
+ __pypackages__/
106
+
107
+ # Celery stuff
108
+ celerybeat-schedule
109
+ celerybeat.pid
110
+
111
+ # SageMath parsed files
112
+ *.sage.py
113
+
114
+ # Environments
115
+ .env
116
+ .venv
117
+ env/
118
+ venv/
119
+ ENV/
120
+ env.bak/
121
+ venv.bak/
122
+
123
+ # Spyder project settings
124
+ .spyderproject
125
+ .spyproject
126
+
127
+ # Rope project settings
128
+ .ropeproject
129
+
130
+ # mkdocs documentation
131
+ /site
132
+
133
+ # mypy
134
+ .mypy_cache/
135
+ .dmypy.json
136
+ dmypy.json
137
+
138
+ # Pyre type checker
139
+ .pyre/
140
+
141
+ # pytype static type analyzer
142
+ .pytype/
143
+
144
+ # Cython debug symbols
145
+ cython_debug/
README.md ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ # Welcome to Streamlit!
2
+
3
+ Edit `/streamlit_app.py` to customize this app to your heart's desire :heart:
4
+
5
+ If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
6
+ forums](https://discuss.streamlit.io).
pages/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ from .search_engine import page as search_engine_page
2
+ from .document import page as document_page
pages/document.py ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import re
3
+ import json
4
+ import datetime
5
+ import itertools
6
+ import requests
7
+ from PIL import Image
8
+ import base64
9
+ import streamlit as st
10
+
11
+ def page():
12
+ record = st.session_state.get("selected_record")
13
+ st.set_page_config(
14
+ page_title=f"Record {record['filename']}",
15
+ page_icon="👨‍⚕️",
16
+ layout="wide",
17
+ initial_sidebar_state="collapsed",
18
+ )
19
+ st.button("Back", on_click=lambda: set_record(None))
20
+
21
+ st.write(record)
pages/search_engine.py ADDED
@@ -0,0 +1,166 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import re
3
+ import json
4
+ import datetime
5
+ import itertools
6
+ import requests
7
+ from PIL import Image
8
+ import base64
9
+ import streamlit as st
10
+ from huggingface_hub import ModelSearchArguments
11
+ import webbrowser
12
+ from numerize.numerize import numerize
13
+
14
+ def paginator(label, articles, articles_per_page=10, on_sidebar=True):
15
+ # https://gist.github.com/treuille/2ce0acb6697f205e44e3e0f576e810b7
16
+ """Lets the user paginate a set of article.
17
+ Parameters
18
+ ----------
19
+ label : str
20
+ The label to display over the pagination widget.
21
+ article : Iterator[Any]
22
+ The articles to display in the paginator.
23
+ articles_per_page: int
24
+ The number of articles to display per page.
25
+ on_sidebar: bool
26
+ Whether to display the paginator widget on the sidebar.
27
+
28
+ Returns
29
+ -------
30
+ Iterator[Tuple[int, Any]]
31
+ An iterator over *only the article on that page*, including
32
+ the item's index.
33
+ """
34
+
35
+ # Figure out where to display the paginator
36
+ if on_sidebar:
37
+ location = st.sidebar.empty()
38
+ else:
39
+ location = st.empty()
40
+
41
+ # Display a pagination selectbox in the specified location.
42
+ articles = list(articles)
43
+ n_pages = (len(articles) - 1) // articles_per_page + 1
44
+ page_format_func = lambda i: f"Results {i*10} to {i*10 +10 -1}"
45
+ page_number = location.selectbox(label, range(n_pages), format_func=page_format_func)
46
+
47
+ # Iterate over the articles in the page to let the user display them.
48
+ min_index = page_number * articles_per_page
49
+ max_index = min_index + articles_per_page
50
+
51
+ return itertools.islice(enumerate(articles), min_index, max_index)
52
+
53
+
54
+ def page():
55
+ st.set_page_config(
56
+ page_title="HF Search Engine",
57
+ page_icon="🔎",
58
+ layout="wide",
59
+ initial_sidebar_state="auto",
60
+ # menu_items={
61
+ # "Get Help": "https://www.extremelycoolapp.com/help",
62
+ # "Report a bug": "https://www.extremelycoolapp.com/bug",
63
+ # "About": "# This is a header. This is an *extremely* cool app!",
64
+ # },
65
+ )
66
+
67
+ ### SIDEBAR
68
+ search_backend = st.sidebar.selectbox(
69
+ "Search Engine",
70
+ ["hfapi", "custom"],
71
+ format_func=lambda x: {"hfapi": "Huggingface API", "custom": "Sentence Bert"}[x],
72
+ )
73
+ limit_results = st.sidebar.number_input("Limit results", min_value=0, value=10)
74
+
75
+ st.sidebar.markdown("# Filters")
76
+ args = ModelSearchArguments()
77
+ library = st.sidebar.multiselect(
78
+ "Library", args.library.values(), format_func=lambda x: {v: k for k, v in args.library.items()}[x]
79
+ )
80
+ task = st.sidebar.multiselect(
81
+ "Task", args.pipeline_tag.values(), format_func=lambda x: {v: k for k, v in args.pipeline_tag.items()}[x]
82
+ )
83
+
84
+ ### MAIN PAGE
85
+ st.markdown(
86
+ "<h1 style='text-align: center; '>🔎🤗 HF Search Engine</h1>",
87
+ unsafe_allow_html=True,
88
+ )
89
+
90
+ # Search bar
91
+ search_query = st.text_input(
92
+ "Search for a model in HuggingFace", value="", max_chars=None, key=None, type="default"
93
+ )
94
+
95
+ # Search API
96
+ endpoint = "http://localhost:5000"
97
+ headers = {
98
+ "Content-Type": "application/json",
99
+ "api-key": "password",
100
+ }
101
+ search_url = f"{endpoint}/{search_backend}/search"
102
+ filters = {
103
+ "library": library,
104
+ "task": task,
105
+ }
106
+ search_body = {
107
+ "query": search_query,
108
+ "filters": json.dumps(filters, default=str),
109
+ "limit": limit_results,
110
+ }
111
+
112
+ if search_query != "":
113
+ response = requests.post(search_url, headers=headers, json=search_body).json()
114
+
115
+ record_list = []
116
+ _ = [
117
+ record_list.append(
118
+ {
119
+ "modelId": record["modelId"],
120
+ "tags": record["tags"],
121
+ "downloads": record["downloads"],
122
+ "likes": record["likes"],
123
+ }
124
+ )
125
+ for record in response.get("value")
126
+ ]
127
+
128
+ # filter results
129
+
130
+ if record_list:
131
+ st.write(f'Search results ({response.get("count")}):')
132
+
133
+ if response.get("count") > 100:
134
+ shown_results = 100
135
+ else:
136
+ shown_results = response.get("count")
137
+
138
+ for i, record in paginator(
139
+ f"Select results (showing {shown_results} of {response.get('count')} results)",
140
+ record_list,
141
+ ):
142
+ col1, col2, col3 = st.columns([5,1,1])
143
+ col1.metric("Model", record["modelId"])
144
+ col2.metric("N° downloads", numerize(record["downloads"]))
145
+ col3.metric("N° likes", numerize(record["likes"]))
146
+ st.button(f"View model", on_click=lambda record=record: webbrowser.open(f"https://huggingface.co/{record['modelId']})"), key=record["modelId"])
147
+ st.markdown(f"**Tags:** {' • '.join(record['tags'])}")
148
+
149
+ # TODO: embed huggingface spaces
150
+ # import streamlit.components.v1 as components
151
+ # components.html(
152
+ # f"""
153
+ # <link rel="stylesheet" href="https://gradio.s3-us-west-2.amazonaws.com/2.6.2/static/bundle.css">
154
+ # <div id="target"></div>
155
+ # <script src="https://gradio.s3-us-west-2.amazonaws.com/2.6.2/static/bundle.js"></script>
156
+ # <script>
157
+ # launchGradioFromSpaces("abidlabs/question-answering", "#target")
158
+ # </script>
159
+ # """,
160
+ # height=400,
161
+ # )
162
+
163
+ st.markdown("---")
164
+
165
+ else:
166
+ st.write(f"No Search results, please try again with different keywords")
requirements.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ altair
2
+ pandas
3
+ streamlit
4
+ huggingface_hub
5
+ numerize
server/api.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from flask import Flask, request
2
+ import json
3
+ from huggingface_hub import HfApi, ModelFilter, DatasetFilter, ModelSearchArguments
4
+ from pprint import pprint
5
+
6
+ app = Flask(__name__)
7
+
8
+
9
+ @app.route("/hello")
10
+ def hello():
11
+ return "<h1 style='color:blue'>Hello There!</h1>"
12
+
13
+
14
+ @app.route("/hfapi/search", methods=["POST"])
15
+ def hf_api():
16
+ request_data = request.get_json()
17
+ query = request_data.get("query")
18
+ filters = json.loads(request_data.get("filters"))
19
+ limit = request_data.get("limit", 5)
20
+ print("query", query)
21
+ print("filters", filters)
22
+ print("limit", limit)
23
+
24
+ api = HfApi()
25
+ filt = ModelFilter(
26
+ task=filters["task"],
27
+ library=filters["library"],
28
+ )
29
+ models = api.list_models(search=query, filter=filt, limit=limit, full=True)
30
+ res = []
31
+ for model in models:
32
+ model = model.__dict__
33
+ res.append(
34
+ {
35
+ "modelId": model.get("modelId"),
36
+ "tags": model.get("tags"),
37
+ "downloads": model.get("downloads"),
38
+ "likes": model.get("likes"),
39
+ }
40
+ )
41
+ count = len(res)
42
+ if len(res) > limit:
43
+ res = res[:limit]
44
+ pprint(res)
45
+ return json.dumps({"value": res, "count": count})
46
+
47
+
48
+ @app.route("/custom/search", methods=["POST"])
49
+ def main():
50
+ request_data = request.get_json()
51
+ query = request_data.get("query")
52
+ filters = json.loads(request_data.get("filters"))
53
+ limit = request_data.get("limit", 5)
54
+ print("query", query)
55
+ print("filters", filters)
56
+ print("limit", limit)
57
+
58
+ # records, count_filtered = search_query(query=request_data["query"], filters=filters, top=request_data["top"])
59
+ # assert len(set([record["id"] for record in records])) == len(records), "ids of results are not unique"
60
+ # res = {"value": records, "count": count_filtered}
61
+ # return json.dumps(res, indent=2)
62
+
63
+
64
+ if __name__ == "__main__":
65
+ app.run(host="localhost", port=5000)
streamlit_app.py ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ from pages import search_engine_page, document_page
3
+
4
+
5
+ if "selected_record" not in st.session_state:
6
+ st.session_state["selected_record"] = None
7
+
8
+
9
+ def set_record(record):
10
+ st.session_state["selected_record"] = record
11
+
12
+
13
+ if not st.session_state["selected_record"]: # search engine page
14
+ search_engine_page()
15
+
16
+ else: # a record has been selected
17
+ document_page()
18
+
19
+
20
+ st.markdown(
21
+ """<style>
22
+ a:link , a:visited{
23
+ color: blue;
24
+ background-color: transparent;
25
+ text-decoration: underline;
26
+ }
27
+
28
+ a:hover, a:active {
29
+ color: red;
30
+ background-color: transparent;
31
+ text-decoration: underline;
32
+ }
33
+
34
+ .footer {
35
+ # position: fixed;
36
+ left: 0;
37
+ bottom: 0;
38
+ width: 100%;
39
+ background-color: white;
40
+ color: black;
41
+ text-align: center;
42
+ }
43
+ </style>
44
+ <div class="footer">
45
+ <p>Made with ❤️ by <b>Nouamane Tazi</b></p>
46
+ </div>
47
+ """,
48
+ unsafe_allow_html=True,
49
+ )