Spaces:
Runtime error
Runtime error
File size: 5,111 Bytes
fa926f8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 |
"""DeepPhys - 2D Convolutional Attention Network.
DeepPhys: Video-Based Physiological Measurement Using Convolutional Attention Networks
ECCV, 2018
Weixuan Chen, Daniel McDuff
"""
import torch
import torch.nn as nn
class Attention_mask(nn.Module):
def __init__(self):
super(Attention_mask, self).__init__()
def forward(self, x):
xsum = torch.sum(x, dim=2, keepdim=True)
xsum = torch.sum(xsum, dim=3, keepdim=True)
xshape = tuple(x.size())
return x / xsum * xshape[2] * xshape[3] * 0.5
def get_config(self):
"""May be generated manually. """
config = super(Attention_mask, self).get_config()
return config
class DeepPhys(nn.Module):
def __init__(self, in_channels=3, nb_filters1=32, nb_filters2=64, kernel_size=3, dropout_rate1=0.25,
dropout_rate2=0.5, pool_size=(2, 2), nb_dense=128, img_size=36):
"""Definition of DeepPhys.
Args:
in_channels: the number of input channel. Default: 3
img_size: height/width of each frame. Default: 36.
Returns:
DeepPhys model.
"""
super(DeepPhys, self).__init__()
self.in_channels = in_channels
self.kernel_size = kernel_size
self.dropout_rate1 = dropout_rate1
self.dropout_rate2 = dropout_rate2
self.pool_size = pool_size
self.nb_filters1 = nb_filters1
self.nb_filters2 = nb_filters2
self.nb_dense = nb_dense
# Motion branch convs
self.motion_conv1 = nn.Conv2d(self.in_channels, self.nb_filters1, kernel_size=self.kernel_size, padding=(1, 1),
bias=True)
self.motion_conv2 = nn.Conv2d(self.nb_filters1, self.nb_filters1, kernel_size=self.kernel_size, bias=True)
self.motion_conv3 = nn.Conv2d(self.nb_filters1, self.nb_filters2, kernel_size=self.kernel_size, padding=(1, 1),
bias=True)
self.motion_conv4 = nn.Conv2d(self.nb_filters2, self.nb_filters2, kernel_size=self.kernel_size, bias=True)
# Apperance branch convs
self.apperance_conv1 = nn.Conv2d(self.in_channels, self.nb_filters1, kernel_size=self.kernel_size,
padding=(1, 1), bias=True)
self.apperance_conv2 = nn.Conv2d(self.nb_filters1, self.nb_filters1, kernel_size=self.kernel_size, bias=True)
self.apperance_conv3 = nn.Conv2d(self.nb_filters1, self.nb_filters2, kernel_size=self.kernel_size,
padding=(1, 1), bias=True)
self.apperance_conv4 = nn.Conv2d(self.nb_filters2, self.nb_filters2, kernel_size=self.kernel_size, bias=True)
# Attention layers
self.apperance_att_conv1 = nn.Conv2d(self.nb_filters1, 1, kernel_size=1, padding=(0, 0), bias=True)
self.attn_mask_1 = Attention_mask()
self.apperance_att_conv2 = nn.Conv2d(self.nb_filters2, 1, kernel_size=1, padding=(0, 0), bias=True)
self.attn_mask_2 = Attention_mask()
# Avg pooling
self.avg_pooling_1 = nn.AvgPool2d(self.pool_size)
self.avg_pooling_2 = nn.AvgPool2d(self.pool_size)
self.avg_pooling_3 = nn.AvgPool2d(self.pool_size)
# Dropout layers
self.dropout_1 = nn.Dropout(self.dropout_rate1)
self.dropout_2 = nn.Dropout(self.dropout_rate1)
self.dropout_3 = nn.Dropout(self.dropout_rate1)
self.dropout_4 = nn.Dropout(self.dropout_rate2)
# Dense layers
if img_size == 36:
self.final_dense_1 = nn.Linear(3136, self.nb_dense, bias=True)
elif img_size == 72:
self.final_dense_1 = nn.Linear(16384, self.nb_dense, bias=True)
elif img_size == 96:
self.final_dense_1 = nn.Linear(30976, self.nb_dense, bias=True)
else:
raise Exception('Unsupported image size')
self.final_dense_2 = nn.Linear(self.nb_dense, 1, bias=True)
def forward(self, inputs, params=None):
diff_input = inputs[:, :3, :, :]
raw_input = inputs[:, 3:, :, :]
d1 = torch.tanh(self.motion_conv1(diff_input))
d2 = torch.tanh(self.motion_conv2(d1))
r1 = torch.tanh(self.apperance_conv1(raw_input))
r2 = torch.tanh(self.apperance_conv2(r1))
g1 = torch.sigmoid(self.apperance_att_conv1(r2))
g1 = self.attn_mask_1(g1)
gated1 = d2 * g1
d3 = self.avg_pooling_1(gated1)
d4 = self.dropout_1(d3)
r3 = self.avg_pooling_2(r2)
r4 = self.dropout_2(r3)
d5 = torch.tanh(self.motion_conv3(d4))
d6 = torch.tanh(self.motion_conv4(d5))
r5 = torch.tanh(self.apperance_conv3(r4))
r6 = torch.tanh(self.apperance_conv4(r5))
g2 = torch.sigmoid(self.apperance_att_conv2(r6))
g2 = self.attn_mask_2(g2)
gated2 = d6 * g2
d7 = self.avg_pooling_3(gated2)
d8 = self.dropout_3(d7)
d9 = d8.reshape(d8.size(0), -1)
d10 = torch.tanh(self.final_dense_1(d9))
d11 = self.dropout_4(d10)
out = self.final_dense_2(d11)
return out
|