File size: 7,671 Bytes
e0c7c25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
import os
import numpy
from scipy.interpolate import RectBivariateSpline

def activation_visualization(image, data, level, alpha=0.5, source_shape=None,
        crop=False, zoom=None, border=2, negate=False, return_mask=False,
        **kwargs):
    """
    Makes a visualiztion image of activation data overlaid on the image.
    Params:
        image The original image.
        data The single channel feature map.
        alpha The darkening to apply in inactive regions of the image.
        level The threshold of activation levels to highlight.
    """
    if len(image.shape) == 2:
        # Puff up grayscale image to RGB.
        image = image[:,:,None] * numpy.array([[[1, 1, 1]]])
    surface = activation_surface(data, target_shape=image.shape[:2],
            source_shape=source_shape, **kwargs)
    if negate:
        surface = -surface
        level = -level
    if crop:
        # crop to source_shape
        if source_shape is not None:
            ch, cw = ((t - s) // 2 for s, t in zip(
                source_shape, image.shape[:2]))
            image = image[ch:ch+source_shape[0], cw:cw+source_shape[1]]
            surface = surface[ch:ch+source_shape[0], cw:cw+source_shape[1]]
        if crop is True:
            crop = surface.shape
        elif not hasattr(crop, '__len__'):
            crop = (crop, crop)
        if zoom is not None:
            source_rect = best_sub_rect(surface >= level, crop, zoom,
                    pad=border)
        else:
            source_rect = (0, surface.shape[0], 0, surface.shape[1])
        image = zoom_image(image, source_rect, crop)
        surface = zoom_image(surface, source_rect, crop)
    mask = (surface >= level)
    # Add a yellow border at the edge of the mask for contrast
    result = (mask[:, :, None] * (1 - alpha) + alpha) * image
    if border:
        edge = mask_border(mask)[:,:,None]
        result = numpy.maximum(edge * numpy.array([[[200, 200, 0]]]), result)
    if not return_mask:
        return result
    mask_image = (1 - mask[:, :, None]) * numpy.array(
            [[[0, 0, 0, 255 * (1 - alpha)]]], dtype=numpy.uint8)
    if border:
        mask_image = numpy.maximum(edge * numpy.array([[[200, 200, 0, 255]]]),
                mask_image)
    return result, mask_image

def activation_surface(data, target_shape=None, source_shape=None,
        scale_offset=None, deg=1, pad=True):
    """
    Generates an upsampled activation sample.
    Params:
        target_shape Shape of the output array.
        source_shape The centered shape of the output to match with data
            when upscaling.  Defaults to the whole target_shape.
        scale_offset The amount by which to scale, then offset data
            dimensions to end up with target dimensions.  A pair of pairs.
        deg Degree of interpolation to apply (1 = linear, etc).
        pad True to zero-pad the edge instead of doing a funny edge interp.
    """
    # Default is that nothing is resized.
    if target_shape is None:
        target_shape = data.shape
    # Make a default scale_offset to fill the image if there isn't one
    if scale_offset is None:
        scale = tuple(float(ts) / ds
                for ts, ds in zip(target_shape, data.shape))
        offset = tuple(0.5 * s - 0.5 for s in scale)
    else:
        scale, offset = (v for v in zip(*scale_offset))
    # Now we adjust offsets to take into account cropping and so on
    if source_shape is not None:
        offset = tuple(o + (ts - ss) / 2.0
                for o, ss, ts in zip(offset, source_shape, target_shape))
    # Pad the edge with zeros for sensible edge behavior
    if pad:
        zeropad = numpy.zeros(
                (data.shape[0] + 2, data.shape[1] + 2), dtype=data.dtype)
        zeropad[1:-1, 1:-1] = data
        data = zeropad
        offset = tuple((o - s) for o, s in zip(offset, scale))
    # Upsample linearly
    ty, tx = (numpy.arange(ts) for ts in target_shape)
    sy, sx = (numpy.arange(ss) * s + o
            for ss, s, o in zip(data.shape, scale, offset))
    levels = RectBivariateSpline(
            sy, sx, data, kx=deg, ky=deg)(ty, tx, grid=True)
    # Return the mask.
    return levels

def mask_border(mask, border=2):
    """Given a mask computes a border mask"""
    from scipy import ndimage
    struct = ndimage.generate_binary_structure(2, 2)
    erosion = numpy.ones((mask.shape[0] + 10, mask.shape[1] + 10), dtype='int')
    erosion[5:5+mask.shape[0], 5:5+mask.shape[1]] = ~mask
    for _ in range(border):
        erosion = ndimage.binary_erosion(erosion, struct)
    return ~mask ^ erosion[5:5+mask.shape[0], 5:5+mask.shape[1]]

def bounding_rect(mask, pad=0):
    """Returns (r, b, l, r) boundaries so that all nonzero pixels in mask
    have locations (i, j) with  t <= i < b, and l <= j < r."""
    nz = mask.nonzero()
    if len(nz[0]) == 0:
        # print('no pixels')
        return (0, mask.shape[0], 0, mask.shape[1])
    (t, b), (l, r) = [(max(0, p.min() - pad), min(s, p.max() + 1 + pad))
            for p, s in zip(nz, mask.shape)]
    return (t, b, l, r)

def best_sub_rect(mask, shape, max_zoom=None, pad=2):
    """Finds the smallest subrectangle containing all the nonzeros of mask,
    matching the aspect ratio of shape, and where the zoom-up ratio is no
    more than max_zoom"""
    t, b, l, r = bounding_rect(mask, pad=pad)
    height = max(b - t, int(round(float(shape[0]) * (r - l) / shape[1])))
    if max_zoom is not None:
        height = int(max(round(float(shape[0]) / max_zoom), height))
    width = int(round(float(shape[1]) * height / shape[0]))
    nt = min(mask.shape[0] - height, max(0, (b + t - height) // 2))
    nb = nt + height
    nl = min(mask.shape[1] - width, max(0, (r + l - width) // 2))
    nr = nl + width
    return (nt, nb, nl, nr)

def zoom_image(img, source_rect, target_shape=None):
    """Zooms pixels from the source_rect of img to target_shape."""
    import warnings
    from scipy.ndimage import zoom
    if target_shape is None:
        target_shape = img.shape
    st, sb, sl, sr = source_rect
    source = img[st:sb, sl:sr]
    if source.shape == target_shape:
        return source
    zoom_tuple = tuple(float(t) / s
            for t, s in zip(target_shape, source.shape[:2])
            ) + (1,) * (img.ndim - 2)
    with warnings.catch_warnings():
        warnings.simplefilter('ignore', UserWarning) # "output shape of zoom"
        target = zoom(source, zoom_tuple)
    assert target.shape[:2] == target_shape, (target.shape, target_shape)
    return target

def scale_offset(dilations):
    if len(dilations) == 0:
        return (1, 0)
    scale, offset = scale_offset(dilations[1:])
    kernel, stride, padding = dilations[0]
    scale *= stride
    offset *= stride
    offset += (kernel - 1) / 2.0 - padding
    return scale, offset

def choose_level(feature_map, percentile=0.8):
    '''
    Chooses the top 80% level (or whatever the level chosen).
    '''
    data_range = numpy.sort(feature_map.flatten())
    return numpy.interp(
            percentile, numpy.linspace(0, 1, len(data_range)), data_range)

def dilations(modulelist):
    result = []
    for module in modulelist:
        settings = tuple(getattr(module, n, d)
            for n, d in (('kernel_size', 1), ('stride', 1), ('padding', 0)))
        settings = (((s, s) if not isinstance(s, tuple) else s)
            for s in settings)
        if settings != ((1, 1), (1, 1), (0, 0)):
            result.append(zip(*settings))
    return zip(*result)

def grid_scale_offset(modulelist):
    '''Returns (yscale, yoffset), (xscale, xoffset) given a list of modules'''
    return tuple(scale_offset(d) for d in dilations(modulelist))