Spaces:
Sleeping
Sleeping
Anon4review
commited on
Commit
•
c9a3acd
1
Parent(s):
75f4ede
initial commit
Browse files- app.py +310 -0
- model.pt +3 -0
- requirements.txt +7 -0
- vision_transformer.py +330 -0
app.py
ADDED
@@ -0,0 +1,310 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import gradio as gr
|
2 |
+
import torch
|
3 |
+
import os
|
4 |
+
import sys
|
5 |
+
import cv2
|
6 |
+
import matplotlib
|
7 |
+
import matplotlib.pyplot as plt
|
8 |
+
import numpy as np
|
9 |
+
from PIL import Image
|
10 |
+
from PIL import ImageFont
|
11 |
+
from PIL import ImageDraw
|
12 |
+
from scipy.stats import rankdata
|
13 |
+
|
14 |
+
import torch
|
15 |
+
import torch.nn as nn
|
16 |
+
import torchvision
|
17 |
+
from torchvision import transforms as pth_transforms
|
18 |
+
import torchvision.transforms as transforms
|
19 |
+
from einops import rearrange, repeat
|
20 |
+
import vision_transformer as vits
|
21 |
+
|
22 |
+
def get_vit256(pretrained_weights, arch='vit_small', device=torch.device('cpu')):
|
23 |
+
r"""
|
24 |
+
Builds ViT-256 Model.
|
25 |
+
|
26 |
+
Args:
|
27 |
+
- pretrained_weights (str): Path to ViT-256 Model Checkpoint.
|
28 |
+
- arch (str): Which model architecture.
|
29 |
+
- device (torch): Torch device to save model.
|
30 |
+
|
31 |
+
Returns:
|
32 |
+
- model256 (torch.nn): Initialized model.
|
33 |
+
"""
|
34 |
+
|
35 |
+
checkpoint_key = 'teacher'
|
36 |
+
device = torch.device("cpu") if torch.cuda.is_available() else torch.device("cpu")
|
37 |
+
model256 = vits.__dict__[arch](patch_size=16, num_classes=0)
|
38 |
+
for p in model256.parameters():
|
39 |
+
p.requires_grad = False
|
40 |
+
model256.eval()
|
41 |
+
model256.to(device)
|
42 |
+
|
43 |
+
if os.path.isfile(pretrained_weights):
|
44 |
+
state_dict = torch.load(pretrained_weights, map_location="cpu")
|
45 |
+
if checkpoint_key is not None and checkpoint_key in state_dict:
|
46 |
+
print(f"Take key {checkpoint_key} in provided checkpoint dict")
|
47 |
+
state_dict = state_dict[checkpoint_key]
|
48 |
+
# remove `module.` prefix
|
49 |
+
state_dict = {k.replace("module.", ""): v for k, v in state_dict.items()}
|
50 |
+
# remove `backbone.` prefix induced by multicrop wrapper
|
51 |
+
state_dict = {k.replace("backbone.", ""): v for k, v in state_dict.items()}
|
52 |
+
msg = model256.load_state_dict(state_dict, strict=False)
|
53 |
+
print('Pretrained weights found at {} and loaded with msg: {}'.format(pretrained_weights, msg))
|
54 |
+
return model256
|
55 |
+
|
56 |
+
def cmap_map(function, cmap):
|
57 |
+
r"""
|
58 |
+
Applies function (which should operate on vectors of shape 3: [r, g, b]), on colormap cmap.
|
59 |
+
This routine will break any discontinuous points in a colormap.
|
60 |
+
|
61 |
+
Args:
|
62 |
+
- function (function)
|
63 |
+
- cmap (matplotlib.colormap)
|
64 |
+
|
65 |
+
Returns:
|
66 |
+
- matplotlib.colormap
|
67 |
+
"""
|
68 |
+
cdict = cmap._segmentdata
|
69 |
+
step_dict = {}
|
70 |
+
# Firt get the list of points where the segments start or end
|
71 |
+
for key in ('red', 'green', 'blue'):
|
72 |
+
step_dict[key] = list(map(lambda x: x[0], cdict[key]))
|
73 |
+
step_list = sum(step_dict.values(), [])
|
74 |
+
step_list = np.array(list(set(step_list)))
|
75 |
+
# Then compute the LUT, and apply the function to the LUT
|
76 |
+
reduced_cmap = lambda step : np.array(cmap(step)[0:3])
|
77 |
+
old_LUT = np.array(list(map(reduced_cmap, step_list)))
|
78 |
+
new_LUT = np.array(list(map(function, old_LUT)))
|
79 |
+
# Now try to make a minimal segment definition of the new LUT
|
80 |
+
cdict = {}
|
81 |
+
for i, key in enumerate(['red','green','blue']):
|
82 |
+
this_cdict = {}
|
83 |
+
for j, step in enumerate(step_list):
|
84 |
+
if step in step_dict[key]:
|
85 |
+
this_cdict[step] = new_LUT[j, i]
|
86 |
+
elif new_LUT[j,i] != old_LUT[j, i]:
|
87 |
+
this_cdict[step] = new_LUT[j, i]
|
88 |
+
colorvector = list(map(lambda x: x + (x[1], ), this_cdict.items()))
|
89 |
+
colorvector.sort()
|
90 |
+
cdict[key] = colorvector
|
91 |
+
|
92 |
+
return matplotlib.colors.LinearSegmentedColormap('colormap',cdict,1024)
|
93 |
+
|
94 |
+
|
95 |
+
def identity(x):
|
96 |
+
r"""
|
97 |
+
Identity Function.
|
98 |
+
|
99 |
+
Args:
|
100 |
+
- x:
|
101 |
+
|
102 |
+
Returns:
|
103 |
+
- x
|
104 |
+
"""
|
105 |
+
return x
|
106 |
+
|
107 |
+
def tensorbatch2im(input_image, imtype=np.uint8):
|
108 |
+
r""""
|
109 |
+
Converts a Tensor array into a numpy image array.
|
110 |
+
|
111 |
+
Args:
|
112 |
+
- input_image (torch.Tensor): (B, C, W, H) Torch Tensor.
|
113 |
+
- imtype (type): the desired type of the converted numpy array
|
114 |
+
|
115 |
+
Returns:
|
116 |
+
- image_numpy (np.array): (B, W, H, C) Numpy Array.
|
117 |
+
"""
|
118 |
+
if not isinstance(input_image, np.ndarray):
|
119 |
+
image_numpy = input_image.cpu().float().numpy() # convert it into a numpy array
|
120 |
+
#if image_numpy.shape[0] == 1: # grayscale to RGB
|
121 |
+
# image_numpy = np.tile(image_numpy, (3, 1, 1))
|
122 |
+
image_numpy = (np.transpose(image_numpy, (0, 2, 3, 1)) + 1) / 2.0 * 255.0 # post-processing: tranpose and scaling
|
123 |
+
else: # if it is a numpy array, do nothing
|
124 |
+
image_numpy = input_image
|
125 |
+
return image_numpy.astype(imtype)
|
126 |
+
|
127 |
+
def getConcatImage(imgs, how='horizontal', gap=0):
|
128 |
+
r"""
|
129 |
+
Function to concatenate list of images (vertical or horizontal).
|
130 |
+
|
131 |
+
Args:
|
132 |
+
- imgs (list of PIL.Image): List of PIL Images to concatenate.
|
133 |
+
- how (str): How the images are concatenated (either 'horizontal' or 'vertical')
|
134 |
+
- gap (int): Gap (in px) between images
|
135 |
+
|
136 |
+
Return:
|
137 |
+
- dst (PIL.Image): Concatenated image result.
|
138 |
+
"""
|
139 |
+
gap_dist = (len(imgs)-1)*gap
|
140 |
+
|
141 |
+
if how == 'vertical':
|
142 |
+
w, h = np.max([img.width for img in imgs]), np.sum([img.height for img in imgs])
|
143 |
+
h += gap_dist
|
144 |
+
curr_h = 0
|
145 |
+
dst = Image.new('RGBA', (w, h), color=(255, 255, 255, 0))
|
146 |
+
for img in imgs:
|
147 |
+
dst.paste(img, (0, curr_h))
|
148 |
+
curr_h += img.height + gap
|
149 |
+
|
150 |
+
elif how == 'horizontal':
|
151 |
+
w, h = np.sum([img.width for img in imgs]), np.min([img.height for img in imgs])
|
152 |
+
w += gap_dist
|
153 |
+
curr_w = 0
|
154 |
+
dst = Image.new('RGBA', (w, h), color=(255, 255, 255, 0))
|
155 |
+
|
156 |
+
for idx, img in enumerate(imgs):
|
157 |
+
dst.paste(img, (curr_w, 0))
|
158 |
+
curr_w += img.width + gap
|
159 |
+
|
160 |
+
return dst
|
161 |
+
|
162 |
+
|
163 |
+
def add_margin(pil_img, top, right, bottom, left, color):
|
164 |
+
r"""
|
165 |
+
Adds custom margin to PIL.Image.
|
166 |
+
"""
|
167 |
+
width, height = pil_img.size
|
168 |
+
new_width = width + right + left
|
169 |
+
new_height = height + top + bottom
|
170 |
+
result = Image.new(pil_img.mode, (new_width, new_height), color)
|
171 |
+
result.paste(pil_img, (left, top))
|
172 |
+
return result
|
173 |
+
|
174 |
+
|
175 |
+
def concat_scores256(attns, size=(256,256)):
|
176 |
+
r"""
|
177 |
+
"""
|
178 |
+
rank = lambda v: rankdata(v)*100/len(v)
|
179 |
+
color_block = [rank(attn.flatten()).reshape(size) for attn in attns]
|
180 |
+
color_hm = np.concatenate([
|
181 |
+
np.concatenate(color_block[i:(i+16)], axis=1)
|
182 |
+
for i in range(0,256,16)
|
183 |
+
])
|
184 |
+
return color_hm
|
185 |
+
|
186 |
+
|
187 |
+
|
188 |
+
def get_scores256(attns, size=(256,256)):
|
189 |
+
r"""
|
190 |
+
"""
|
191 |
+
rank = lambda v: rankdata(v)*100/len(v)
|
192 |
+
color_block = [rank(attn.flatten()).reshape(size) for attn in attns][0]
|
193 |
+
return color_block
|
194 |
+
|
195 |
+
|
196 |
+
def get_patch_attention_scores(patch, model256, scale=1, device256=torch.device('cpu')):
|
197 |
+
t = transforms.Compose([
|
198 |
+
transforms.ToTensor(),
|
199 |
+
transforms.Normalize(
|
200 |
+
[0.5, 0.5, 0.5], [0.5, 0.5, 0.5]
|
201 |
+
)
|
202 |
+
])
|
203 |
+
|
204 |
+
with torch.no_grad():
|
205 |
+
batch_256 = t(patch).unsqueeze(0)
|
206 |
+
batch_256 = batch_256.to(device256, non_blocking=True)
|
207 |
+
features_256 = model256(batch_256)
|
208 |
+
|
209 |
+
attention_256 = model256.get_last_selfattention(batch_256)
|
210 |
+
nh = attention_256.shape[1] # number of head
|
211 |
+
attention_256 = attention_256[:, :, 0, 1:].reshape(256, nh, -1)
|
212 |
+
attention_256 = attention_256.reshape(1, nh, 16, 16)
|
213 |
+
attention_256 = nn.functional.interpolate(attention_256, scale_factor=int(16/scale), mode="nearest").cpu().numpy()
|
214 |
+
|
215 |
+
if scale != 1:
|
216 |
+
batch_256 = nn.functional.interpolate(batch_256, scale_factor=(1/scale), mode="nearest")
|
217 |
+
|
218 |
+
return tensorbatch2im(batch_256), attention_256
|
219 |
+
|
220 |
+
|
221 |
+
def create_patch_heatmaps_concat(patch, model256, output_dir=None, fname=None, threshold=None,
|
222 |
+
offset=16, alpha=0.5, cmap=plt.get_cmap('coolwarm')):
|
223 |
+
r"""
|
224 |
+
Creates patch heatmaps (concatenated for easy comparison)
|
225 |
+
|
226 |
+
Args:
|
227 |
+
- patch (PIL.Image): 256 x 256 Image
|
228 |
+
- model256 (torch.nn): 256-Level ViT
|
229 |
+
- output_dir (str): Save directory / subdirectory
|
230 |
+
- fname (str): Naming structure of files
|
231 |
+
- offset (int): How much to offset (from top-left corner with zero-padding) the region by for blending
|
232 |
+
- alpha (float): Image blending factor for cv2.addWeighted
|
233 |
+
- cmap (matplotlib.pyplot): Colormap for creating heatmaps
|
234 |
+
|
235 |
+
Returns:
|
236 |
+
- None
|
237 |
+
"""
|
238 |
+
patch1 = patch.copy()
|
239 |
+
patch2 = add_margin(patch.crop((16,16,256,256)), top=0, left=0, bottom=16, right=16, color=(255,255,255))
|
240 |
+
b256_1, a256_1 = get_patch_attention_scores(patch1, model256)
|
241 |
+
b256_1, a256_2 = get_patch_attention_scores(patch2, model256)
|
242 |
+
save_region = np.array(patch.copy())
|
243 |
+
s = 256
|
244 |
+
offset_2 = offset
|
245 |
+
|
246 |
+
if threshold != None:
|
247 |
+
ths = []
|
248 |
+
for i in range(6):
|
249 |
+
score256_1 = get_scores256(a256_1[:,i,:,:], size=(s,)*2)
|
250 |
+
score256_2 = get_scores256(a256_2[:,i,:,:], size=(s,)*2)
|
251 |
+
new_score256_2 = np.zeros_like(score256_2)
|
252 |
+
new_score256_2[offset_2:s, offset_2:s] = score256_2[:(s-offset_2), :(s-offset_2)]
|
253 |
+
overlay256 = np.ones_like(score256_2)*100
|
254 |
+
overlay256[offset_2:s, offset_2:s] += 100
|
255 |
+
score256 = (score256_1+new_score256_2)/overlay256
|
256 |
+
|
257 |
+
mask256 = score256.copy()
|
258 |
+
mask256[mask256 < threshold] = 0
|
259 |
+
mask256[mask256 > threshold] = 0.95
|
260 |
+
|
261 |
+
color_block256 = (cmap(mask256)*255)[:,:,:3].astype(np.uint8)
|
262 |
+
region256_hm = cv2.addWeighted(color_block256, alpha, save_region.copy(), 1-alpha, 0, save_region.copy())
|
263 |
+
region256_hm[mask256==0] = 0
|
264 |
+
img_inverse = save_region.copy()
|
265 |
+
img_inverse[mask256 == 0.95] = 0
|
266 |
+
ths.append(region256_hm+img_inverse)
|
267 |
+
|
268 |
+
ths = [Image.fromarray(img) for img in ths]
|
269 |
+
|
270 |
+
getConcatImage([getConcatImage(ths[0:3]),
|
271 |
+
getConcatImage(ths[4:6])], how='vertical').save(os.path.join(output_dir, '%s_256th.png' % (fname)))
|
272 |
+
|
273 |
+
|
274 |
+
hms = []
|
275 |
+
for i in range(6):
|
276 |
+
score256_1 = get_scores256(a256_1[:,i,:,:], size=(s,)*2)
|
277 |
+
score256_2 = get_scores256(a256_2[:,i,:,:], size=(s,)*2)
|
278 |
+
new_score256_2 = np.zeros_like(score256_2)
|
279 |
+
new_score256_2[offset_2:s, offset_2:s] = score256_2[:(s-offset_2), :(s-offset_2)]
|
280 |
+
overlay256 = np.ones_like(score256_2)*100
|
281 |
+
overlay256[offset_2:s, offset_2:s] += 100
|
282 |
+
score256 = (score256_1+new_score256_2)/overlay256
|
283 |
+
color_block256 = (cmap(score256)*255)[:,:,:3].astype(np.uint8)
|
284 |
+
region256_hm = cv2.addWeighted(color_block256, alpha, save_region.copy(), 1-alpha, 0, save_region.copy())
|
285 |
+
hms.append(region256_hm)
|
286 |
+
|
287 |
+
hms = [Image.fromarray(img) for img in hms]
|
288 |
+
return getConcatImage([getConcatImage(hms[0:3], how='horizontal', gap=10),
|
289 |
+
getConcatImage(hms[4:6], how='horizontal', gap=10)], how='vertical', gap=10)
|
290 |
+
|
291 |
+
def demo_patch_heatmaps(input_image):
|
292 |
+
light_jet = cmap_map(lambda x: x/2 + 0.5, matplotlib.cm.jet)
|
293 |
+
model256 = get_vit256(pretrained_weights=pretrained_weights256)
|
294 |
+
demo_heatmap = create_patch_heatmaps_concat(input_image, model256, cmap=light_jet)
|
295 |
+
return demo_heatmap
|
296 |
+
|
297 |
+
|
298 |
+
pretrained_weights256 = './model.pt'
|
299 |
+
|
300 |
+
title = "Demo for 11604"
|
301 |
+
description = "To use, upload a 256 x 256 patch (20X magnification). \
|
302 |
+
The output will generate attention results from 6 attention heads."
|
303 |
+
|
304 |
+
iface = gr.Interface(fn=demo_patch_heatmaps,
|
305 |
+
inputs=gr.inputs.Image(type='pil'),
|
306 |
+
outputs="image",
|
307 |
+
title=title,
|
308 |
+
description=description,
|
309 |
+
allow_flagging=False)
|
310 |
+
iface.launch()
|
model.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:d04e3718649a13a5a49ae5c274ae3aefb9deb229ef5194106bb8ecbfd4f00c61
|
3 |
+
size 704238867
|
requirements.txt
ADDED
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
torch
|
2 |
+
torchvision
|
3 |
+
pillow
|
4 |
+
numpy
|
5 |
+
scipy
|
6 |
+
einops
|
7 |
+
opencv-python-headless
|
vision_transformer.py
ADDED
@@ -0,0 +1,330 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Copyright (c) Facebook, Inc. and its affiliates.
|
2 |
+
#
|
3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4 |
+
# you may not use this file except in compliance with the License.
|
5 |
+
# You may obtain a copy of the License at
|
6 |
+
#
|
7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8 |
+
#
|
9 |
+
# Unless required by applicable law or agreed to in writing, software
|
10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12 |
+
# See the License for the specific language governing permissions and
|
13 |
+
# limitations under the License.
|
14 |
+
"""
|
15 |
+
Mostly copy-paste from timm library.
|
16 |
+
https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py
|
17 |
+
"""
|
18 |
+
import math
|
19 |
+
from functools import partial
|
20 |
+
|
21 |
+
import torch
|
22 |
+
import torch.nn as nn
|
23 |
+
|
24 |
+
|
25 |
+
def _no_grad_trunc_normal_(tensor, mean, std, a, b):
|
26 |
+
# Cut & paste from PyTorch official master until it's in a few official releases - RW
|
27 |
+
# Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
|
28 |
+
def norm_cdf(x):
|
29 |
+
# Computes standard normal cumulative distribution function
|
30 |
+
return (1. + math.erf(x / math.sqrt(2.))) / 2.
|
31 |
+
|
32 |
+
if (mean < a - 2 * std) or (mean > b + 2 * std):
|
33 |
+
warnings.warn("mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
|
34 |
+
"The distribution of values may be incorrect.",
|
35 |
+
stacklevel=2)
|
36 |
+
|
37 |
+
with torch.no_grad():
|
38 |
+
# Values are generated by using a truncated uniform distribution and
|
39 |
+
# then using the inverse CDF for the normal distribution.
|
40 |
+
# Get upper and lower cdf values
|
41 |
+
l = norm_cdf((a - mean) / std)
|
42 |
+
u = norm_cdf((b - mean) / std)
|
43 |
+
|
44 |
+
# Uniformly fill tensor with values from [l, u], then translate to
|
45 |
+
# [2l-1, 2u-1].
|
46 |
+
tensor.uniform_(2 * l - 1, 2 * u - 1)
|
47 |
+
|
48 |
+
# Use inverse cdf transform for normal distribution to get truncated
|
49 |
+
# standard normal
|
50 |
+
tensor.erfinv_()
|
51 |
+
|
52 |
+
# Transform to proper mean, std
|
53 |
+
tensor.mul_(std * math.sqrt(2.))
|
54 |
+
tensor.add_(mean)
|
55 |
+
|
56 |
+
# Clamp to ensure it's in the proper range
|
57 |
+
tensor.clamp_(min=a, max=b)
|
58 |
+
return tensor
|
59 |
+
|
60 |
+
|
61 |
+
def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):
|
62 |
+
# type: (Tensor, float, float, float, float) -> Tensor
|
63 |
+
return _no_grad_trunc_normal_(tensor, mean, std, a, b)
|
64 |
+
|
65 |
+
|
66 |
+
def drop_path(x, drop_prob: float = 0., training: bool = False):
|
67 |
+
if drop_prob == 0. or not training:
|
68 |
+
return x
|
69 |
+
keep_prob = 1 - drop_prob
|
70 |
+
shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets
|
71 |
+
random_tensor = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device)
|
72 |
+
random_tensor.floor_() # binarize
|
73 |
+
output = x.div(keep_prob) * random_tensor
|
74 |
+
return output
|
75 |
+
|
76 |
+
|
77 |
+
class DropPath(nn.Module):
|
78 |
+
"""Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).
|
79 |
+
"""
|
80 |
+
def __init__(self, drop_prob=None):
|
81 |
+
super(DropPath, self).__init__()
|
82 |
+
self.drop_prob = drop_prob
|
83 |
+
|
84 |
+
def forward(self, x):
|
85 |
+
return drop_path(x, self.drop_prob, self.training)
|
86 |
+
|
87 |
+
|
88 |
+
class Mlp(nn.Module):
|
89 |
+
def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):
|
90 |
+
super().__init__()
|
91 |
+
out_features = out_features or in_features
|
92 |
+
hidden_features = hidden_features or in_features
|
93 |
+
self.fc1 = nn.Linear(in_features, hidden_features)
|
94 |
+
self.act = act_layer()
|
95 |
+
self.fc2 = nn.Linear(hidden_features, out_features)
|
96 |
+
self.drop = nn.Dropout(drop)
|
97 |
+
|
98 |
+
def forward(self, x):
|
99 |
+
x = self.fc1(x)
|
100 |
+
x = self.act(x)
|
101 |
+
x = self.drop(x)
|
102 |
+
x = self.fc2(x)
|
103 |
+
x = self.drop(x)
|
104 |
+
return x
|
105 |
+
|
106 |
+
|
107 |
+
class Attention(nn.Module):
|
108 |
+
def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.):
|
109 |
+
super().__init__()
|
110 |
+
self.num_heads = num_heads
|
111 |
+
head_dim = dim // num_heads
|
112 |
+
self.scale = qk_scale or head_dim ** -0.5
|
113 |
+
|
114 |
+
self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
|
115 |
+
self.attn_drop = nn.Dropout(attn_drop)
|
116 |
+
self.proj = nn.Linear(dim, dim)
|
117 |
+
self.proj_drop = nn.Dropout(proj_drop)
|
118 |
+
|
119 |
+
def forward(self, x):
|
120 |
+
B, N, C = x.shape
|
121 |
+
qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
|
122 |
+
q, k, v = qkv[0], qkv[1], qkv[2]
|
123 |
+
|
124 |
+
attn = (q @ k.transpose(-2, -1)) * self.scale
|
125 |
+
attn = attn.softmax(dim=-1)
|
126 |
+
attn = self.attn_drop(attn)
|
127 |
+
|
128 |
+
x = (attn @ v).transpose(1, 2).reshape(B, N, C)
|
129 |
+
x = self.proj(x)
|
130 |
+
x = self.proj_drop(x)
|
131 |
+
return x, attn
|
132 |
+
|
133 |
+
|
134 |
+
class Block(nn.Module):
|
135 |
+
def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
|
136 |
+
drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm):
|
137 |
+
super().__init__()
|
138 |
+
self.norm1 = norm_layer(dim)
|
139 |
+
self.attn = Attention(
|
140 |
+
dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)
|
141 |
+
self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
|
142 |
+
self.norm2 = norm_layer(dim)
|
143 |
+
mlp_hidden_dim = int(dim * mlp_ratio)
|
144 |
+
self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
|
145 |
+
|
146 |
+
def forward(self, x, return_attention=False):
|
147 |
+
y, attn = self.attn(self.norm1(x))
|
148 |
+
if return_attention:
|
149 |
+
return attn
|
150 |
+
x = x + self.drop_path(y)
|
151 |
+
x = x + self.drop_path(self.mlp(self.norm2(x)))
|
152 |
+
return x
|
153 |
+
|
154 |
+
|
155 |
+
class PatchEmbed(nn.Module):
|
156 |
+
""" Image to Patch Embedding
|
157 |
+
"""
|
158 |
+
def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768):
|
159 |
+
super().__init__()
|
160 |
+
num_patches = (img_size // patch_size) * (img_size // patch_size)
|
161 |
+
self.img_size = img_size
|
162 |
+
self.patch_size = patch_size
|
163 |
+
self.num_patches = num_patches
|
164 |
+
|
165 |
+
self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
|
166 |
+
|
167 |
+
def forward(self, x):
|
168 |
+
B, C, H, W = x.shape
|
169 |
+
x = self.proj(x).flatten(2).transpose(1, 2)
|
170 |
+
return x
|
171 |
+
|
172 |
+
|
173 |
+
class VisionTransformer(nn.Module):
|
174 |
+
""" Vision Transformer """
|
175 |
+
def __init__(self, img_size=[224], patch_size=16, in_chans=3, num_classes=0, embed_dim=768, depth=12,
|
176 |
+
num_heads=12, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0.,
|
177 |
+
drop_path_rate=0., norm_layer=nn.LayerNorm, **kwargs):
|
178 |
+
super().__init__()
|
179 |
+
self.num_features = self.embed_dim = embed_dim
|
180 |
+
|
181 |
+
self.patch_embed = PatchEmbed(
|
182 |
+
img_size=img_size[0], patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
|
183 |
+
num_patches = self.patch_embed.num_patches
|
184 |
+
|
185 |
+
self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
|
186 |
+
self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))
|
187 |
+
self.pos_drop = nn.Dropout(p=drop_rate)
|
188 |
+
|
189 |
+
dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule
|
190 |
+
self.blocks = nn.ModuleList([
|
191 |
+
Block(
|
192 |
+
dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,
|
193 |
+
drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer)
|
194 |
+
for i in range(depth)])
|
195 |
+
self.norm = norm_layer(embed_dim)
|
196 |
+
|
197 |
+
# Classifier head
|
198 |
+
self.head = nn.Linear(embed_dim, num_classes) if num_classes > 0 else nn.Identity()
|
199 |
+
|
200 |
+
trunc_normal_(self.pos_embed, std=.02)
|
201 |
+
trunc_normal_(self.cls_token, std=.02)
|
202 |
+
self.apply(self._init_weights)
|
203 |
+
|
204 |
+
def _init_weights(self, m):
|
205 |
+
if isinstance(m, nn.Linear):
|
206 |
+
trunc_normal_(m.weight, std=.02)
|
207 |
+
if isinstance(m, nn.Linear) and m.bias is not None:
|
208 |
+
nn.init.constant_(m.bias, 0)
|
209 |
+
elif isinstance(m, nn.LayerNorm):
|
210 |
+
nn.init.constant_(m.bias, 0)
|
211 |
+
nn.init.constant_(m.weight, 1.0)
|
212 |
+
|
213 |
+
def interpolate_pos_encoding(self, x, w, h):
|
214 |
+
npatch = x.shape[1] - 1
|
215 |
+
N = self.pos_embed.shape[1] - 1
|
216 |
+
if npatch == N and w == h:
|
217 |
+
return self.pos_embed
|
218 |
+
class_pos_embed = self.pos_embed[:, 0]
|
219 |
+
patch_pos_embed = self.pos_embed[:, 1:]
|
220 |
+
dim = x.shape[-1]
|
221 |
+
w0 = w // self.patch_embed.patch_size
|
222 |
+
h0 = h // self.patch_embed.patch_size
|
223 |
+
# we add a small number to avoid floating point error in the interpolation
|
224 |
+
# see discussion at https://github.com/facebookresearch/dino/issues/8
|
225 |
+
w0, h0 = w0 + 0.1, h0 + 0.1
|
226 |
+
patch_pos_embed = nn.functional.interpolate(
|
227 |
+
patch_pos_embed.reshape(1, int(math.sqrt(N)), int(math.sqrt(N)), dim).permute(0, 3, 1, 2),
|
228 |
+
scale_factor=(w0 / math.sqrt(N), h0 / math.sqrt(N)),
|
229 |
+
mode='bicubic',
|
230 |
+
)
|
231 |
+
assert int(w0) == patch_pos_embed.shape[-2] and int(h0) == patch_pos_embed.shape[-1]
|
232 |
+
patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
|
233 |
+
return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1)
|
234 |
+
|
235 |
+
def prepare_tokens(self, x):
|
236 |
+
B, nc, w, h = x.shape
|
237 |
+
x = self.patch_embed(x) # patch linear embedding
|
238 |
+
|
239 |
+
# add the [CLS] token to the embed patch tokens
|
240 |
+
cls_tokens = self.cls_token.expand(B, -1, -1)
|
241 |
+
x = torch.cat((cls_tokens, x), dim=1)
|
242 |
+
|
243 |
+
# add positional encoding to each token
|
244 |
+
x = x + self.interpolate_pos_encoding(x, w, h)
|
245 |
+
|
246 |
+
return self.pos_drop(x)
|
247 |
+
|
248 |
+
def forward(self, x):
|
249 |
+
x = self.prepare_tokens(x)
|
250 |
+
for blk in self.blocks:
|
251 |
+
x = blk(x)
|
252 |
+
x = self.norm(x)
|
253 |
+
return x[:, 0]
|
254 |
+
|
255 |
+
def get_last_selfattention(self, x):
|
256 |
+
x = self.prepare_tokens(x)
|
257 |
+
for i, blk in enumerate(self.blocks):
|
258 |
+
if i < len(self.blocks) - 1:
|
259 |
+
x = blk(x)
|
260 |
+
else:
|
261 |
+
# return attention of the last block
|
262 |
+
return blk(x, return_attention=True)
|
263 |
+
|
264 |
+
def get_intermediate_layers(self, x, n=1):
|
265 |
+
x = self.prepare_tokens(x)
|
266 |
+
# we return the output tokens from the `n` last blocks
|
267 |
+
output = []
|
268 |
+
for i, blk in enumerate(self.blocks):
|
269 |
+
x = blk(x)
|
270 |
+
if len(self.blocks) - i <= n:
|
271 |
+
output.append(self.norm(x))
|
272 |
+
return output
|
273 |
+
|
274 |
+
|
275 |
+
def vit_tiny(patch_size=16, **kwargs):
|
276 |
+
model = VisionTransformer(
|
277 |
+
patch_size=patch_size, embed_dim=192, depth=12, num_heads=3, mlp_ratio=4,
|
278 |
+
qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
|
279 |
+
return model
|
280 |
+
|
281 |
+
|
282 |
+
def vit_small(patch_size=16, **kwargs):
|
283 |
+
model = VisionTransformer(
|
284 |
+
patch_size=patch_size, embed_dim=384, depth=12, num_heads=6, mlp_ratio=4,
|
285 |
+
qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
|
286 |
+
return model
|
287 |
+
|
288 |
+
|
289 |
+
def vit_base(patch_size=16, **kwargs):
|
290 |
+
model = VisionTransformer(
|
291 |
+
patch_size=patch_size, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4,
|
292 |
+
qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs)
|
293 |
+
return model
|
294 |
+
|
295 |
+
|
296 |
+
class DINOHead(nn.Module):
|
297 |
+
def __init__(self, in_dim, out_dim, use_bn=False, norm_last_layer=True, nlayers=3, hidden_dim=2048, bottleneck_dim=256):
|
298 |
+
super().__init__()
|
299 |
+
nlayers = max(nlayers, 1)
|
300 |
+
if nlayers == 1:
|
301 |
+
self.mlp = nn.Linear(in_dim, bottleneck_dim)
|
302 |
+
else:
|
303 |
+
layers = [nn.Linear(in_dim, hidden_dim)]
|
304 |
+
if use_bn:
|
305 |
+
layers.append(nn.BatchNorm1d(hidden_dim))
|
306 |
+
layers.append(nn.GELU())
|
307 |
+
for _ in range(nlayers - 2):
|
308 |
+
layers.append(nn.Linear(hidden_dim, hidden_dim))
|
309 |
+
if use_bn:
|
310 |
+
layers.append(nn.BatchNorm1d(hidden_dim))
|
311 |
+
layers.append(nn.GELU())
|
312 |
+
layers.append(nn.Linear(hidden_dim, bottleneck_dim))
|
313 |
+
self.mlp = nn.Sequential(*layers)
|
314 |
+
self.apply(self._init_weights)
|
315 |
+
self.last_layer = nn.utils.weight_norm(nn.Linear(bottleneck_dim, out_dim, bias=False))
|
316 |
+
self.last_layer.weight_g.data.fill_(1)
|
317 |
+
if norm_last_layer:
|
318 |
+
self.last_layer.weight_g.requires_grad = False
|
319 |
+
|
320 |
+
def _init_weights(self, m):
|
321 |
+
if isinstance(m, nn.Linear):
|
322 |
+
trunc_normal_(m.weight, std=.02)
|
323 |
+
if isinstance(m, nn.Linear) and m.bias is not None:
|
324 |
+
nn.init.constant_(m.bias, 0)
|
325 |
+
|
326 |
+
def forward(self, x):
|
327 |
+
x = self.mlp(x)
|
328 |
+
x = nn.functional.normalize(x, dim=-1, p=2)
|
329 |
+
x = self.last_layer(x)
|
330 |
+
return x
|