paulbricman commited on
Commit
bd587b1
1 Parent(s): 084a5f0

feat: implement navigator, viewport, inspector

Browse files
.streamlit/config.toml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
1
+ [theme]
2
+ base="light"
3
+ primaryColor="#228b22"
4
+ font="monospace"
components/__pycache__/base.cpython-38.pyc ADDED
Binary file (690 Bytes). View file
components/__pycache__/core.cpython-38.pyc ADDED
Binary file (626 Bytes). View file
components/__pycache__/inspector.cpython-38.pyc ADDED
Binary file (1.44 kB). View file
components/__pycache__/metadata.cpython-38.pyc ADDED
Binary file (1.44 kB). View file
components/__pycache__/navigator.cpython-38.pyc ADDED
Binary file (1.37 kB). View file
components/__pycache__/viewport.cpython-38.pyc ADDED
Binary file (1.35 kB). View file
components/core.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+
3
+
4
+ def header_section():
5
+ # st.markdown('### 💡 conceptarium')
6
+ pass
7
+
8
+ def footer_section():
9
+ hide_streamlit_style = '''
10
+ <style>
11
+ #MainMenu {visibility: hidden;}
12
+ footer {visibility: hidden;}
13
+ </style>
14
+ '''
15
+ st.markdown(hide_streamlit_style, unsafe_allow_html=True)
components/inspector.py ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ import pickle
3
+ from sentence_transformers import util
4
+ from datetime import datetime
5
+
6
+
7
+ def get_name():
8
+ return '🔍 inspector'
9
+
10
+
11
+ @st.cache(persist=True, allow_output_mutation=True)
12
+ def load_thoughts():
13
+ return pickle.load(open('conceptarium/metadata.pickle', 'rb'))
14
+
15
+
16
+ def paint():
17
+ if st.session_state.get('navigator_embedding', None) is not None:
18
+ thoughts = load_thoughts()
19
+ results = util.semantic_search(
20
+ [st.session_state['navigator_embedding']],
21
+ [e.embedding for e in thoughts])
22
+
23
+ if results[0][0]['score'] > 0.98:
24
+ thought = thoughts[results[0][0]['corpus_id']]
25
+ st.markdown('**modality**: ' + thought.modality)
26
+ st.markdown('**filename**: ' + thought.filename.split('/')[-1])
27
+ st.markdown('**timestamp**: ' + datetime.utcfromtimestamp(int(thought.timestamp)).strftime("%d.%m.%Y"))
28
+ st.markdown('**interest**: ' + str(thought.interest))
29
+
30
+ else:
31
+ st.caption('No thought within reach.')
32
+
33
+
components/navigator.py ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ from sentence_transformers import SentenceTransformer
3
+ import io
4
+ from PIL import Image
5
+
6
+
7
+ def get_name():
8
+ return '🧭 navigator'
9
+
10
+
11
+ @st.cache(persist=True, allow_output_mutation=True)
12
+ def load_model():
13
+ return SentenceTransformer('clip-ViT-B-32')
14
+
15
+
16
+ def paint():
17
+ model = load_model()
18
+ modality = st.selectbox('modality', ['language', 'imagery'],
19
+ ['language', 'imagery'].index(st.session_state.get('navigator_modality', 'language')))
20
+ embedding = None
21
+
22
+ if modality == 'language':
23
+ if st.session_state.get('navigator_modality', None) == 'language':
24
+ value = st.session_state.get('navigator_input', '')
25
+ else:
26
+ value = ''
27
+
28
+ input = st.text_area('input', value=value, height=300)
29
+ embedding = model.encode(input, convert_to_tensor=True, normalize_embeddings=True)
30
+ elif modality == 'imagery':
31
+ if st.session_state.get('navigator_modality', None) == 'imagery':
32
+ value = st.session_state['navigator_input']
33
+ elif st.session_state.get('navigator_modality', None) == 'language':
34
+ value = None
35
+
36
+ input = st.file_uploader('input')
37
+
38
+ if input is not None:
39
+ embedding = model.encode(Image.open(io.BytesIO(input.getvalue())), convert_to_tensor=True, normalize_embeddings=True)
40
+ elif value is not None:
41
+ embedding = st.session_state['navigator_embedding']
42
+
43
+ if st.button('jump'):
44
+ st.session_state['navigator_modality'] = modality
45
+ st.session_state['navigator_embedding'] = embedding
46
+ st.session_state['navigator_input'] = input
47
+
48
+
components/viewport.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ import pickle
3
+ from sentence_transformers import util
4
+
5
+
6
+ def get_name():
7
+ return '🪟 viewport'
8
+
9
+
10
+ @st.cache(persist=True, allow_output_mutation=True)
11
+ def load_thoughts():
12
+ return pickle.load(open('conceptarium/metadata.pickle', 'rb'))
13
+
14
+
15
+ def paint():
16
+ if st.session_state.get('navigator_embedding', None) is not None:
17
+ thoughts = load_thoughts()
18
+ results = util.semantic_search(
19
+ [st.session_state['navigator_embedding']],
20
+ [e.embedding for e in thoughts])
21
+
22
+ for result in results[0]:
23
+ thought = thoughts[result['corpus_id']]
24
+ if thought.modality == 'language':
25
+ content = open(thought.filename).read()
26
+ st.success(content)
27
+ elif thought.modality == 'imagery':
28
+ content = open(thought.filename, 'rb').read()
29
+ st.image(content)
30
+
31
+ if st.button('jump', thought):
32
+ st.session_state['navigator_input'] = content
33
+ st.session_state['navigator_modality'] = thought.modality
34
+ st.session_state['navigator_embedding'] = thought.embedding
35
+ st.experimental_rerun()
36
+
37
+
docs.png DELETED
Binary file (85.4 kB)
main.py CHANGED
@@ -1,128 +1,21 @@
1
- from fastapi import FastAPI, File, BackgroundTasks
2
- from fastapi.datastructures import UploadFile
3
- from fastapi.staticfiles import StaticFiles
4
- from fastapi.responses import HTMLResponse, PlainTextResponse, FileResponse
5
- import secrets
6
- from pathlib import Path
7
- from typing import Optional
8
- from PIL import Image
9
- import io
10
 
11
- from util import *
12
- from responses import *
 
13
 
14
- app = FastAPI()
15
- model = load_model()
16
- init()
17
 
18
- app.mount('/conceptarium', StaticFiles(directory='conceptarium'))
19
- app.mount('/assets', StaticFiles(directory='assets'))
20
 
 
 
21
 
22
- @app.get('/save/lang/form')
23
- async def save_language_form():
24
- return HTMLResponse(save_lang_form_response())
25
-
26
-
27
- @app.get('/save/imag/form')
28
- async def save_imagery_form():
29
- return HTMLResponse(save_imag_form_response())
30
-
31
-
32
- @app.get('/save/lang')
33
- async def save_language(content: str, background_tasks: BackgroundTasks):
34
- if len(content) > 0:
35
- filename = 'conceptarium/' + secrets.token_urlsafe(8) + '.txt'
36
- open(filename, 'w').write(content)
37
- background_tasks.add_task(save, Thought(filename, content, model))
38
- return HTMLResponse(save_success_response())
39
-
40
-
41
- @app.post('/save/imag')
42
- async def save_imagery(file: UploadFile = File(...)):
43
- content = await file.read()
44
- image = Image.open(io.BytesIO(content)).convert('RGB')
45
- filename = 'conceptarium/' + secrets.token_urlsafe(8) + '.jpg'
46
- image.save(filename, quality=50)
47
- save(Thought(filename, content, model))
48
- return HTMLResponse(save_success_response())
49
-
50
-
51
- @app.get('/find/lang/form')
52
- async def find_by_language_form():
53
- return HTMLResponse(find_lang_form_response())
54
-
55
-
56
- @app.get('/find/lang/html')
57
- async def find_by_language_return_html(content: str, relatedness: Optional[float] = 1, serendipity: Optional[float] = 0, noise: Optional[float] = 0.01, silent: Optional[bool] = False, top_k: Optional[int] = 50):
58
- thoughts = find(content, model, relatedness,
59
- serendipity, noise, silent, top_k)
60
- return HTMLResponse(html_response(thoughts))
61
-
62
-
63
- @app.get('/find/lang/text')
64
- async def find_by_language_return_plaintext(content: str, relatedness: Optional[float] = 1, serendipity: Optional[float] = 0, noise: Optional[float] = 0.01, silent: Optional[bool] = False, top_k: Optional[int] = 50):
65
- thoughts = find(content, model, relatedness,
66
- serendipity, noise, silent, top_k)
67
- return PlainTextResponse(plaintext_response(thoughts))
68
-
69
-
70
- @app.get('/find/lang/file')
71
- async def find_by_language_return_file(content: str, relatedness: Optional[float] = 1, serendipity: Optional[float] = 0, noise: Optional[float] = 0.01, silent: Optional[bool] = False, top_k: Optional[int] = 50):
72
- thoughts = find(content, model, relatedness,
73
- serendipity, noise, silent, top_k)
74
- return FileResponse(file_response(thoughts))
75
-
76
-
77
- @app.get('/find/lang/json')
78
- async def find_by_language_return_json(content: str, relatedness: Optional[float] = 1, serendipity: Optional[float] = 0, noise: Optional[float] = 0.01, silent: Optional[bool] = False, top_k: Optional[int] = 50):
79
- thoughts = find(content, model, relatedness,
80
- serendipity, noise, silent, top_k)
81
- return json_response(thoughts)
82
-
83
-
84
- @app.get('/find/imag/form')
85
- async def find_by_imagery_form():
86
- return HTMLResponse(find_imag_form_response())
87
-
88
-
89
- @app.post('/find/imag/html')
90
- async def find_by_imagery_return_html(file: UploadFile = File(...), relatedness: Optional[float] = 1, serendipity: Optional[float] = 0, noise: Optional[float] = 0.01, silent: Optional[bool] = False, top_k: Optional[int] = 50):
91
- content = await file.read()
92
- thoughts = find(content, model, relatedness,
93
- serendipity, noise, silent, top_k)
94
- return HTMLResponse(html_response(thoughts))
95
-
96
-
97
- @app.post('/find/imag/text')
98
- async def find_by_imagery_return_plaintext(file: UploadFile = File(...), relatedness: Optional[float] = 1, serendipity: Optional[float] = 0, noise: Optional[float] = 0.01, silent: Optional[bool] = False, top_k: Optional[int] = 50):
99
- content = await file.read()
100
- thoughts = find(content, model, relatedness,
101
- serendipity, noise, silent, top_k)
102
- return PlainTextResponse(plaintext_response(thoughts))
103
-
104
-
105
- @app.post('/find/imag/file')
106
- async def find_by_imagery_return_file(file: UploadFile = File(...), relatedness: Optional[float] = 1, serendipity: Optional[float] = 0, noise: Optional[float] = 0.01, silent: Optional[bool] = False, top_k: Optional[int] = 50):
107
- content = await file.read()
108
- thoughts = find(content, model, relatedness,
109
- serendipity, noise, silent, top_k)
110
- return FileResponse(file_response(thoughts))
111
-
112
-
113
- @app.post('/find/imag/json')
114
- async def find_by_imagery_return_json(file: UploadFile = File(...), relatedness: Optional[float] = 1, serendipity: Optional[float] = 0, noise: Optional[float] = 0.01, silent: Optional[bool] = False, top_k: Optional[int] = 50):
115
- content = await file.read()
116
- thoughts = find(content, model, relatedness,
117
- serendipity, noise, silent, top_k)
118
- return json_response(thoughts)
119
-
120
-
121
- @app.get('/rset/embs')
122
- async def reset_embeddings_handle():
123
- reset_embeddings(model)
124
-
125
-
126
- @app.get('/dump')
127
- async def dump_conceptarium():
128
- return FileResponse(archive_response(), filename='conceptarium.zip')
1
+ import streamlit as st
2
+ from components import core, inspector, navigator, viewport
3
+ import streamlit.components.v1 as components
 
 
 
 
 
 
4
 
5
+ st.set_page_config(
6
+ page_title='💡 conceptarium',
7
+ layout='wide')
8
 
9
+ top = st.empty()
 
 
10
 
11
+ core.header_section()
12
+ core.footer_section()
13
 
14
+ cols = st.columns([1, 1, 1, 1])
15
+ layout = [[navigator], [viewport], [], [inspector]]
16
 
17
+ for col_idx, col in enumerate(layout):
18
+ for component in col:
19
+ with cols[col_idx]:
20
+ with st.expander(component.get_name(), True):
21
+ component.paint()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
requirements.txt DELETED
@@ -1,8 +0,0 @@
1
- sentence_transformers
2
- numpy
3
- fastapi
4
- Pillow
5
- secrets
6
- aiofiles
7
- python-multipart
8
- pytest
 
 
 
 
 
 
 
 
responses.py DELETED
@@ -1,92 +0,0 @@
1
- from util import get_doc_paths
2
- from zipfile import ZipFile
3
- import numpy as np
4
- import time
5
-
6
-
7
- def html_response(thoughts):
8
- html = '<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">'
9
- html += '<link rel="stylesheet" type="text/css" href="/assets/style.css" media="screen" />'
10
- html += '<script src="//code.jquery.com/jquery-1.10.2.js"></script>'
11
- html += '<script> $(function(){$("#header").load("/assets/header.html");}); </script>'
12
- html += '<div id="header""></div><div class="board">'
13
-
14
- for thought in thoughts:
15
- html += '<div class="card">'
16
- if thought.modality == 'language':
17
- content = open(thought.filename, 'r').read()
18
- html += '<div class="card-body">' + content + '</div>'
19
- else:
20
- html += '<img class="card-img" src=\"/' + \
21
- thought.filename + '\">'
22
- html += '</div><br/>'
23
-
24
- html += '</div>'
25
- return html
26
-
27
-
28
- def save_success_response():
29
- return open('assets/success.html').read()
30
-
31
-
32
- def save_lang_form_response():
33
- return open('assets/save_lang.html').read()
34
-
35
-
36
- def save_imag_form_response():
37
- return open('assets/save_imag.html').read()
38
-
39
-
40
- def find_lang_form_response():
41
- return open('assets/find_lang.html').read()
42
-
43
-
44
- def find_imag_form_response():
45
- return open('assets/find_imag.html').read()
46
-
47
-
48
- def archive_response():
49
- paths = get_doc_paths('conceptarium')
50
- with ZipFile('conceptarium.zip', 'w') as zip:
51
- for path in paths:
52
- zip.write(path)
53
-
54
- return 'conceptarium.zip'
55
-
56
-
57
- def plaintext_response(thoughts):
58
- plaintext = ''
59
-
60
- for thought in thoughts:
61
- if thought.modality == 'language':
62
- content = open(thought.filename, 'r').read()
63
- plaintext += '\"' + content + '\"\n'
64
-
65
- return plaintext
66
-
67
-
68
- def file_response(thoughts):
69
- for thought in thoughts:
70
- if thought.modality == 'imagery':
71
- return thought.filename
72
-
73
-
74
- def json_response(thoughts):
75
- response_json = []
76
-
77
- for thought in thoughts:
78
- thought_json = {
79
- 'timestamp': thought.timestamp,
80
- 'interest': thought.interest,
81
- 'activation': np.log(thought.interest / (1 - 0.9)) - 0.9 * np.log((time.time() - thought.timestamp) / (3600 * 24) + 0.1),
82
- 'modality': thought.modality,
83
- 'embedding': thought.embedding.tolist(),
84
- }
85
-
86
- if thought.modality == 'language':
87
- thought_json['content'] = open(thought.filename, 'r').read()
88
- else:
89
- thought_json['filename'] = '/' + thought.filename
90
-
91
- response_json += [thought_json]
92
- return response_json
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
test_main.py DELETED
@@ -1,21 +0,0 @@
1
- from fastapi.testclient import TestClient
2
-
3
- from main import app
4
-
5
- client = TestClient(app)
6
-
7
-
8
- def test_read_style():
9
- response = client.get("/assets/style.css")
10
- assert response.status_code == 200
11
-
12
-
13
- def test_save_lang_find_lang_json():
14
- response = client.get("/save/lang?content=hello")
15
- assert response.status_code == 200
16
- response = client.get("/save/lang?content=hey")
17
- assert response.status_code == 200
18
- response = client.get("/save/lang?content=transistor")
19
- assert response.status_code == 200
20
- response = client.get("/find/lang/json?content=transistor")
21
- assert response.json()[0]['content'] == 'transistor'