huangzhii commited on
Commit
1bf0164
1 Parent(s): 7987133
.gitattributes CHANGED
@@ -32,3 +32,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
32
  *.zip filter=lfs diff=lfs merge=lfs -text
33
  *.zst filter=lfs diff=lfs merge=lfs -text
34
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
32
  *.zip filter=lfs diff=lfs merge=lfs -text
33
  *.zst filter=lfs diff=lfs merge=lfs -text
34
  *tfevents* filter=lfs diff=lfs merge=lfs -text
35
+ *.csv filter=lfs diff=lfs merge=lfs -text
36
+ *.asset filter=lfs diff=lfs merge=lfs -text
37
+ twitter.asset filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ /__pycache__
2
+ *.pyc
app.py CHANGED
@@ -5,8 +5,9 @@ import streamlit as st
5
 
6
 
7
 
 
8
 
9
- st.sidebar.title("Explore our PLIP Demo")
10
 
11
  PAGES = {
12
  "Introduction": home,
 
5
 
6
 
7
 
8
+ #st.set_page_config(layout="wide")
9
 
10
+ st.sidebar.title("Multi-task Vision–Language AI for Pathology")
11
 
12
  PAGES = {
13
  "Introduction": home,
image2image.py CHANGED
@@ -5,7 +5,11 @@ import numpy as np
5
  from PIL import Image
6
  import requests
7
  import tokenizers
 
8
  from io import BytesIO
 
 
 
9
  import torch
10
  from transformers import (
11
  VisionTextDualEncoderModel,
@@ -15,6 +19,7 @@ from transformers import (
15
  AutoProcessor
16
  )
17
  import streamlit.components.v1 as components
 
18
 
19
 
20
  def embed_images(model, images, processor):
@@ -42,51 +47,123 @@ def load_path_clip():
42
  processor = AutoProcessor.from_pretrained("vinid/plip")
43
  return model, processor
44
 
 
 
 
 
 
 
 
 
45
 
46
- def app():
47
- st.title('PLIP Image Search')
48
 
49
- plip_imgURL = pd.read_csv("tweet_eval_retrieval.tsv", sep="\t")
50
- plip_weblink = pd.read_csv("tweet_eval_retrieval_twlnk.tsv", sep="\t")
 
51
 
 
52
  model, processor = load_path_clip()
53
 
54
- image_embedding = load_embeddings("tweet_eval_embeddings.npy")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
 
56
- query = st.file_uploader("Choose a file")
57
 
58
 
 
 
 
 
 
 
59
  if query:
60
  image = Image.open(query)
 
 
 
 
 
 
 
 
61
  single_image = embed_images(model, [image], processor)[0].detach().cpu().numpy()
62
 
63
  single_image = single_image/np.linalg.norm(single_image)
64
 
65
  # Sort IDs by cosine-similarity from high to low
66
  similarity_scores = single_image.dot(image_embedding.T)
67
- id_sorted = np.argsort(similarity_scores)[::-1]
68
 
69
 
70
- best_id = id_sorted[0]
71
- score = similarity_scores[best_id]
72
-
73
- target_weblink = plip_weblink.iloc[best_id]["weblink"]
74
-
75
- st.caption('Most relevant image (similarity = %.4f)' % score)
76
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
77
  components.html('''
78
  <blockquote class="twitter-tweet">
79
  <a href="%s"></a>
80
  </blockquote>
81
  <script async src="https://platform.twitter.com/widgets.js" charset="utf-8">
82
  </script>
83
- ''' % target_weblink,
84
- height=600)
 
85
 
86
 
87
 
88
 
89
 
 
 
90
 
91
 
92
 
 
5
  from PIL import Image
6
  import requests
7
  import tokenizers
8
+ import os
9
  from io import BytesIO
10
+ import pickle
11
+ import base64
12
+
13
  import torch
14
  from transformers import (
15
  VisionTextDualEncoderModel,
 
19
  AutoProcessor
20
  )
21
  import streamlit.components.v1 as components
22
+ from st_clickable_images import clickable_images #pip install st-clickable-images
23
 
24
 
25
  def embed_images(model, images, processor):
 
47
  processor = AutoProcessor.from_pretrained("vinid/plip")
48
  return model, processor
49
 
50
+ def init():
51
+ with open('data/twitter.asset', 'rb') as f:
52
+ data = pickle.load(f)
53
+ meta = data['meta'].reset_index(drop=True)
54
+ image_embedding = data['embedding']
55
+ print(meta.shape, image_embedding.shape)
56
+ validation_subset_index = meta['source'].values == 'Val_Tweets'
57
+ return meta, image_embedding, validation_subset_index
58
 
 
 
59
 
60
+ def app():
61
+ st.title('Image to Image Retrieval')
62
+ st.markdown('#### A pathology image search engine that correlate images with images.')
63
 
64
+ meta, image_embedding, validation_subset_index = init()
65
  model, processor = load_path_clip()
66
 
67
+ st.markdown('Click following examples:')
68
+ example_path = 'data/example_images'
69
+ list_of_examples = [os.path.join(example_path, v) for v in os.listdir(example_path)]
70
+ example_imgs = []
71
+ for file in list_of_examples:
72
+ with open(file, "rb") as image:
73
+ encoded = base64.b64encode(image.read()).decode()
74
+ example_imgs.append(f"data:image/jpeg;base64,{encoded}")
75
+ clicked = clickable_images(
76
+ example_imgs,
77
+ titles=[f"Image #{str(i)}" for i in range(len(example_imgs))],
78
+ div_style={"display": "flex", "justify-content": "center", "flex-wrap": "wrap"},
79
+ img_style={"margin": "5px", "height": "70px"},
80
+ )
81
+ isExampleClicked = False
82
+ if clicked > -1:
83
+ image = Image.open(list_of_examples[clicked])
84
+ isExampleClicked = True
85
+
86
+
87
+
88
+
89
+ data_options = ["All twitter data (2006-03-21 — 2023-01-15)",
90
+ "Twitter validation data (2022-11-16 — 2023-01-15)"]
91
+ st.radio(
92
+ "Or choose dataset for image retrieval 👉",
93
+ key="datapool",
94
+ options=data_options,
95
+ )
96
+
97
 
 
98
 
99
 
100
+ col1, col2 = st.columns(2)
101
+ with col1:
102
+ query = st.file_uploader("Choose a file to upload")
103
+
104
+
105
+ proceed = False
106
  if query:
107
  image = Image.open(query)
108
+ proceed = True
109
+ elif isExampleClicked:
110
+ proceed = True
111
+
112
+ if proceed:
113
+ with col2:
114
+ st.image(image, caption='Your upload')
115
+
116
  single_image = embed_images(model, [image], processor)[0].detach().cpu().numpy()
117
 
118
  single_image = single_image/np.linalg.norm(single_image)
119
 
120
  # Sort IDs by cosine-similarity from high to low
121
  similarity_scores = single_image.dot(image_embedding.T)
 
122
 
123
 
124
+ topn = 5
125
+ if st.session_state.datapool == data_options[0]:
126
+ #Use all twitter data
127
+ id_sorted = np.argsort(similarity_scores)[::-1]
128
+ best_ids = id_sorted[:topn]
129
+ best_scores = similarity_scores[best_ids]
130
+ target_weblinks = meta["weblink"].values[best_ids]
131
+ else:
132
+ #Use validation twitter data
133
+ similarity_scores = similarity_scores[validation_subset_index]
134
+ # Sort IDs by cosine-similarity from high to low
135
+ id_sorted = np.argsort(similarity_scores)[::-1]
136
+ best_ids = id_sorted[:topn]
137
+ best_scores = similarity_scores[best_ids]
138
+ target_weblinks = meta["weblink"].values[validation_subset_index][best_ids]
139
+ #TODO: Avoid duplicated ID
140
+
141
+ topk_options = ['1st', '2nd', '3rd', '4th', '5th']
142
+ st.radio(
143
+ "Choose the most similar 👉",
144
+ key="top_k",
145
+ options=topk_options,
146
+ horizontal=True
147
+ )
148
+ topn_txt = st.session_state.top_k
149
+ topn_value = int(st.session_state.top_k[0])-1
150
+ st.caption(f'The {topn_txt} relevant image (similarity = {best_scores[topn_value]:.4f})')
151
  components.html('''
152
  <blockquote class="twitter-tweet">
153
  <a href="%s"></a>
154
  </blockquote>
155
  <script async src="https://platform.twitter.com/widgets.js" charset="utf-8">
156
  </script>
157
+ ''' % target_weblinks[topn_value],
158
+ height=800)
159
+
160
 
161
 
162
 
163
 
164
 
165
+ st.markdown('Disclaimer')
166
+ st.caption('Please be advised that this function has been developed in compliance with the Twitter policy of data usage and sharing. It is important to note that the results obtained from this function are not intended to constitute medical advice or replace consultation with a qualified medical professional. The use of this function is solely at your own risk and should be consistent with applicable laws, regulations, and ethical considerations. We do not warrant or guarantee the accuracy, completeness, suitability, or usefulness of this function for any particular purpose, and we hereby disclaim any liability arising from any reliance placed on this function or any results obtained from its use. If you wish to review the original Twitter post, you should access the source page directly on Twitter.')
167
 
168
 
169
 
introduction.md CHANGED
@@ -1,2 +1,4 @@
1
 
2
- # Welcome to our PLIP Demo
 
 
 
1
 
2
+ # AI-enabled Multi-task Vision–Language Modeling for Pathology from Large-Scale Public Social Network Knowledge
3
+
4
+ The incomplete understanding of heterogeneous pathology images is limited by the inadequate amount of well-annotated publicly available image–text datasets. In this study, we collected 208,414 well-annotated pathology data. Each has a paired image and text description and this collection is so far the largest public dataset for pathology images. By jointly learning the visual and linguistic representations of the data, we proposed a multi-task AI for pathology, which achieves superior performances across multiple benchmarks and can predict previously unseen data. In addition, this framework allows image retrieval by text inputs. Serving as an image search engine, the ability to retrieve relevant images can be a powerful educational tool. In summary, this large-scale, crowdsourcing, spontaneous, and interactive public social network knowledge enabled us to establish a generic AI for pathology that is capable of handling multiple tasks. This approach has greatly enhanced our understanding and interaction with the enormous amount of pathology data available.
text2image.py CHANGED
@@ -4,6 +4,7 @@ from plip_support import embed_text
4
  import numpy as np
5
  from PIL import Image
6
  import requests
 
7
  import tokenizers
8
  from io import BytesIO
9
  import torch
@@ -45,50 +46,106 @@ def load_path_clip():
45
  processor = AutoProcessor.from_pretrained("vinid/plip")
46
  return model, processor
47
 
 
 
 
 
 
 
 
 
48
 
49
  def app():
50
- st.title('PLIP Image Search')
51
-
52
- plip_imgURL = pd.read_csv("tweet_eval_retrieval.tsv", sep="\t")
53
- plip_weblink = pd.read_csv("tweet_eval_retrieval_twlnk.tsv", sep="\t")
54
 
 
 
 
 
 
55
  model, processor = load_path_clip()
56
 
57
- image_embedding = load_embeddings("tweet_eval_embeddings.npy")
58
-
59
- query = st.text_input('Search Query', '')
60
-
61
- if query:
62
-
63
- text_embedding = embed_texts(model, [query], processor)[0].detach().cpu().numpy()
 
64
 
65
- text_embedding = text_embedding/np.linalg.norm(text_embedding)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
66
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
  # Sort IDs by cosine-similarity from high to low
68
- similarity_scores = text_embedding.dot(image_embedding.T)
69
  id_sorted = np.argsort(similarity_scores)[::-1]
 
 
 
 
70
 
71
-
72
- best_id = id_sorted[0]
73
- score = similarity_scores[best_id]
74
- target_url = plip_imgURL.iloc[best_id]["imageURL"]
75
- target_weblink = plip_weblink.iloc[best_id]["weblink"]
76
-
77
- st.caption('Most relevant image (similarity = %.4f)' % score)
78
- #response = requests.get(target_url)
79
- #img = Image.open(BytesIO(response.content))
80
- #st.image(img)
81
-
82
-
83
- components.html('''
84
- <blockquote class="twitter-tweet">
85
- <a href="%s"></a>
86
- </blockquote>
87
- <script async src="https://platform.twitter.com/widgets.js" charset="utf-8">
88
- </script>
89
- ''' % target_weblink,
90
- height=600)
91
-
92
 
93
 
94
 
@@ -100,6 +157,8 @@ def app():
100
 
101
 
102
 
 
 
103
 
104
 
105
 
 
4
  import numpy as np
5
  from PIL import Image
6
  import requests
7
+ import pickle
8
  import tokenizers
9
  from io import BytesIO
10
  import torch
 
46
  processor = AutoProcessor.from_pretrained("vinid/plip")
47
  return model, processor
48
 
49
+ def init():
50
+ with open('data/twitter.asset', 'rb') as f:
51
+ data = pickle.load(f)
52
+ meta = data['meta'].reset_index(drop=True)
53
+ image_embedding = data['embedding']
54
+ print(meta.shape, image_embedding.shape)
55
+ validation_subset_index = meta['source'].values == 'Val_Tweets'
56
+ return meta, image_embedding, validation_subset_index
57
 
58
  def app():
 
 
 
 
59
 
60
+ st.title('Text to Image Retrieval')
61
+ st.markdown('#### A pathology image search engine that correlate texts directly with images.')
62
+ st.caption('Note: The searching query matches images only. The twitter text does not used for searching.')
63
+
64
+ meta, image_embedding, validation_subset_index = init()
65
  model, processor = load_path_clip()
66
 
67
+ data_options = ["All twitter data (2006-03-21 — 2023-01-15)",
68
+ "Twitter validation data (2022-11-16 — 2023-01-15)"]
69
+ st.radio(
70
+ "Choose dataset for image retrieval 👉",
71
+ key="datapool",
72
+ options=data_options,
73
+ )
74
+
75
 
76
+ col1, col2 = st.columns(2)
77
+ #query = st.text_input('Search Query', '')
78
+ col1_submit = False
79
+ show = False
80
+ with col1:
81
+ # Create selectbox
82
+ examples = ['Breast tumor surrounded by fat',
83
+ 'HER2+ breast tumor',
84
+ 'Colorectal cancer tumor on epithelium',
85
+ 'An image of endometrium epithelium',
86
+ 'Breast cancer DCIS',
87
+ 'Papillary carcinoma in breast tissue',
88
+ ]
89
+ query_1 = st.selectbox("Please select an example query", options=examples)
90
+ #st.info(f":white_check_mark: The written option is {query_1} ")
91
+ col1_submit = True
92
+ show = True
93
 
94
+ with col2:
95
+ form = st.form(key='my_form')
96
+ query_2 = form.text_input(label='Or input your custom query:')
97
+ submit_button = form.form_submit_button(label='Submit')
98
+
99
+ if submit_button:
100
+ col1_submit = False
101
+ show = True
102
+
103
+
104
+ if col1_submit:
105
+ query = query_1
106
+ else:
107
+ query = query_2
108
+
109
+ text_embedding = embed_texts(model, [query], processor)[0].detach().cpu().numpy()
110
+ text_embedding = text_embedding/np.linalg.norm(text_embedding)
111
+
112
+ similarity_scores = text_embedding.dot(image_embedding.T)
113
+
114
+ topn = 5
115
+ if st.session_state.datapool == data_options[0]:
116
+ #Use all twitter data
117
+ id_sorted = np.argsort(similarity_scores)[::-1]
118
+ best_ids = id_sorted[:topn]
119
+ best_scores = similarity_scores[best_ids]
120
+ target_weblinks = meta["weblink"].values[best_ids]
121
+ else:
122
+ #Use validation twitter data
123
+ similarity_scores = similarity_scores[validation_subset_index]
124
  # Sort IDs by cosine-similarity from high to low
 
125
  id_sorted = np.argsort(similarity_scores)[::-1]
126
+ best_ids = id_sorted[:topn]
127
+ best_scores = similarity_scores[best_ids]
128
+ target_weblinks = meta["weblink"].values[validation_subset_index][best_ids]
129
+ #TODO: Avoid duplicated ID
130
 
131
+ topk_options = ['1st', '2nd', '3rd', '4th', '5th']
132
+ st.radio(
133
+ "Choose the most similar 👉",
134
+ key="top_k",
135
+ options=topk_options,
136
+ horizontal=True
137
+ )
138
+ topn_txt = st.session_state.top_k
139
+ topn_value = int(st.session_state.top_k[0])-1
140
+ st.caption(f'The {topn_txt} relevant image (similarity = {best_scores[topn_value]:.4f})')
141
+ components.html('''
142
+ <blockquote class="twitter-tweet">
143
+ <a href="%s"></a>
144
+ </blockquote>
145
+ <script async src="https://platform.twitter.com/widgets.js" charset="utf-8">
146
+ </script>
147
+ ''' % target_weblinks[topn_value],
148
+ height=800)
 
 
 
149
 
150
 
151
 
 
157
 
158
 
159
 
160
+ st.markdown('Disclaimer')
161
+ st.caption('Please be advised that this function has been developed in compliance with the Twitter policy of data usage and sharing. It is important to note that the results obtained from this function are not intended to constitute medical advice or replace consultation with a qualified medical professional. The use of this function is solely at your own risk and should be consistent with applicable laws, regulations, and ethical considerations. We do not warrant or guarantee the accuracy, completeness, suitability, or usefulness of this function for any particular purpose, and we hereby disclaim any liability arising from any reliance placed on this function or any results obtained from its use. If you wish to review the original Twitter post, you should access the source page directly on Twitter.')
162
 
163
 
164
 
tweet_eval_embeddings.npy DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:36e445b069b1d937a0a780ddeab9239df5fd13264e8cd1f6cf033be3210352e1
3
- size 2401408
 
 
 
 
tweet_eval_retrieval.tsv DELETED
The diff for this file is too large to render. See raw diff
 
tweet_eval_retrieval_twlnk.tsv DELETED
The diff for this file is too large to render. See raw diff
 
zeroshot.py DELETED
File without changes