added dsdg without model file

DSDG/DUM/Load_OULUNPUcrop_train.py

@@ -0,0 +1,277 @@
+from __future__ import print_function, division
+import os
+import torch
+import pandas as pd
+import cv2
+import numpy as np
+import random
+import torch
+from torch.utils.data import Dataset, DataLoader
+from torchvision import transforms
+import pdb
+import math
+import os
+import copy
+import imgaug.augmenters as iaa
+
+# face_scale = 1.3  #default for test, for training , can be set from [1.2 to 1.5]
+
+# data augment from 'imgaug' --> Add (value=(-40,40), per_channel=True), GammaContrast (gamma=(0.5,1.5))
+seq = iaa.Sequential([
+    iaa.Add(value=(-40, 40), per_channel=True),  # Add color
+    iaa.GammaContrast(gamma=(0.5, 1.5))  # GammaContrast with a gamma of 0.5 to 1.5
+])
+
+
+# array
+class RandomErasing(object):
+    '''
+    Class that performs Random Erasing in Random Erasing Data Augmentation by Zhong et al. 
+    -------------------------------------------------------------------------------------
+    probability: The probability that the operation will be performed.
+    sl: min erasing area
+    sh: max erasing area
+    r1: min aspect ratio
+    mean: erasing value
+    -------------------------------------------------------------------------------------
+    '''
+
+    def __init__(self, probability=0.5, sl=0.01, sh=0.05, r1=0.5, mean=[0.4914, 0.4822, 0.4465]):
+        self.probability = probability
+        self.mean = mean
+        self.sl = sl
+        self.sh = sh
+        self.r1 = r1
+
+    def __call__(self, sample):
+        img, map_x, spoofing_label = sample['image_x'], sample['map_x'], sample['spoofing_label']
+
+        if random.uniform(0, 1) < self.probability:
+            attempts = np.random.randint(1, 3)
+            for attempt in range(attempts):
+                area = img.shape[0] * img.shape[1]
+
+                target_area = random.uniform(self.sl, self.sh) * area
+                aspect_ratio = random.uniform(self.r1, 1 / self.r1)
+
+                h = int(round(math.sqrt(target_area * aspect_ratio)))
+                w = int(round(math.sqrt(target_area / aspect_ratio)))
+
+                if w < img.shape[1] and h < img.shape[0]:
+                    x1 = random.randint(0, img.shape[0] - h)
+                    y1 = random.randint(0, img.shape[1] - w)
+
+                    img[x1:x1 + h, y1:y1 + w, 0] = self.mean[0]
+                    img[x1:x1 + h, y1:y1 + w, 1] = self.mean[1]
+                    img[x1:x1 + h, y1:y1 + w, 2] = self.mean[2]
+
+        return {'image_x': img, 'map_x': map_x, 'spoofing_label': spoofing_label}
+
+
+# Tensor
+class Cutout(object):
+    def __init__(self, length=50):
+        self.length = length
+
+    def __call__(self, sample):
+        img, map_x, spoofing_label = sample['image_x'], sample['map_x'], sample['spoofing_label']
+        h, w = img.shape[1], img.shape[2]  # Tensor [1][2],  nparray [0][1]
+        mask = np.ones((h, w), np.float32)
+        y = np.random.randint(h)
+        x = np.random.randint(w)
+        length_new = np.random.randint(1, self.length)
+
+        y1 = np.clip(y - length_new // 2, 0, h)
+        y2 = np.clip(y + length_new // 2, 0, h)
+        x1 = np.clip(x - length_new // 2, 0, w)
+        x2 = np.clip(x + length_new // 2, 0, w)
+
+        mask[y1: y2, x1: x2] = 0.
+        mask = torch.from_numpy(mask)
+        mask = mask.expand_as(img)
+        img *= mask
+        return {'image_x': img, 'map_x': map_x, 'spoofing_label': spoofing_label}
+
+
+class Normaliztion(object):
+    """
+        same as mxnet, normalize into [-1, 1]
+        image = (image - 127.5)/128
+    """
+
+    def __call__(self, sample):
+        image_x, map_x, spoofing_label = sample['image_x'], sample['map_x'], sample['spoofing_label']
+        new_image_x = (image_x - 127.5) / 128  # [-1,1]
+        new_map_x = map_x / 255.0  # [0,1]
+        return {'image_x': new_image_x, 'map_x': new_map_x, 'spoofing_label': spoofing_label}
+
+
+class RandomHorizontalFlip(object):
+    """Horizontally flip the given Image randomly with a probability of 0.5."""
+
+    def __call__(self, sample):
+        image_x, map_x, spoofing_label = sample['image_x'], sample['map_x'], sample['spoofing_label']
+
+        new_image_x = np.zeros((256, 256, 3))
+        new_map_x = np.zeros((32, 32))
+
+        p = random.random()
+        if p < 0.5:
+            # print('Flip')
+
+            new_image_x = cv2.flip(image_x, 1)
+            new_map_x = cv2.flip(map_x, 1)
+
+            return {'image_x': new_image_x, 'map_x': new_map_x, 'spoofing_label': spoofing_label}
+        else:
+            # print('no Flip')
+            return {'image_x': image_x, 'map_x': map_x, 'spoofing_label': spoofing_label}
+
+
+class ToTensor(object):
+    """
+        Convert ndarrays in sample to Tensors.
+        process only one batch every time
+    """
+
+    def __call__(self, sample):
+        image_x, map_x, spoofing_label = sample['image_x'], sample['map_x'], sample['spoofing_label']
+
+        # swap color axis because
+        # numpy image: (batch_size) x H x W x C
+        # torch image: (batch_size) x C X H X W
+        image_x = image_x[:, :, ::-1].transpose((2, 0, 1))
+        image_x = np.array(image_x)
+
+        map_x = np.array(map_x)
+
+        spoofing_label_np = np.array([0], dtype=np.long)
+        spoofing_label_np[0] = spoofing_label
+
+        return {'image_x': torch.from_numpy(image_x.astype(np.float)).float(),
+                'map_x': torch.from_numpy(map_x.astype(np.float)).float(),
+                'spoofing_label': torch.from_numpy(spoofing_label_np.astype(np.long)).long()}
+
+
+class Spoofing_train_g(Dataset):
+
+    def __init__(self, info_list, root_dir, map_dir, transform=None):
+
+        # +1,1_1_21_1
+        self.landmarks_frame = pd.read_csv(info_list, delimiter=',', header=None)
+        self.root_dir = root_dir
+        self.map_dir = map_dir
+        self.transform = transform
+
+    def __len__(self):
+        return len(self.landmarks_frame)
+
+    def __getitem__(self, idx):
+        # 1_1_30_1
+        videoname = str(self.landmarks_frame.iloc[idx, 1])
+        image_path = os.path.join(self.root_dir, videoname)
+        map_path = os.path.join(self.map_dir, videoname)
+        image_x, map_x = self.get_single_image_x(image_path, map_path, videoname)
+
+        spoofing_label = self.landmarks_frame.iloc[idx, 0]
+        if spoofing_label == 1:
+            spoofing_label = 1  # real
+        else:
+            spoofing_label = 0
+            map_x = np.zeros((32, 32))  # fake
+
+        sample = {'image_x': image_x, 'map_x': map_x, 'spoofing_label': spoofing_label}
+
+        if self.transform:
+            sample = self.transform(sample)
+        return sample
+
+    def get_idx(self):
+        real_data_idx = []
+        fake_data_idx = []
+        i, j = 0, 0
+        for idx_all in range(self.__len__()):
+            videoname = str(self.landmarks_frame.iloc[idx_all, 1])
+            if videoname[:1] == 'p':
+                fake_data_idx.append(i)
+                i += 1
+            else:
+                real_data_idx.append(j)
+                j += 1
+        return real_data_idx, fake_data_idx
+
+    def get_single_image_x(self, images_path, maps_path, videoname):
+
+        frame_total = len([name for name in os.listdir(images_path) if os.path.isfile(os.path.join(images_path, name))])
+        # random choose 1 frame
+
+        image_id = np.random.randint(1, frame_total)
+
+        if videoname[:1] == 'p':
+            image_id = np.random.randint(1, 100)
+            s = "%d_scene" % image_id
+            image_name = s + '.jpg'
+            # /home/shejiahui5/notespace/data/oulu_img/train_bbox_files/p2_0_1_30/21_scence.jpg
+            s = "%d_depth1D" % image_id
+            map_name = s + '.jpg'
+        else:
+            image_id = np.random.randint(1, frame_total)
+            s = "_%d_scene" % image_id
+            image_name = videoname + s + '.jpg'
+            s = "_%d_depth1D" % image_id
+            map_name = videoname + s + '.jpg'
+
+        image_path = os.path.join(images_path, image_name)
+        map_path = os.path.join(maps_path, map_name)
+
+        map_x = np.zeros((32, 32))
+
+        # RGB
+        image_x = cv2.imread(image_path)
+        image_x = cv2.resize(image_x, (256, 256))
+        # data augment from 'imgaug' --> Add (value=(-40,40), per_channel=True), GammaContrast (gamma=(0.5,1.5))
+        image_x_aug = seq.augment_image(image_x)
+
+        # gray-map
+        if os.path.exists(map_path):
+            map_x = cv2.imread(map_path, 0)
+            map_x = cv2.resize(map_x, (32, 32))
+
+        return image_x_aug, map_x
+
+
+class SeparateBatchSampler(object):
+    def __init__(self, real_data_idx, fake_data_idx, batch_size, ratio, put_back=False):
+        self.batch_size = batch_size
+        self.ratio = ratio
+        self.real_data_num = len(real_data_idx)
+        self.fake_data_num = len(fake_data_idx)
+        self.max_num_image = max(self.real_data_num, self.fake_data_num)
+
+        self.real_data_idx = real_data_idx
+        self.fake_data_idx = fake_data_idx
+
+        self.processed_idx = copy.deepcopy(self.real_data_idx)
+
+    def __len__(self):
+        return self.max_num_image // (int(self.batch_size * self.ratio))
+
+    def __iter__(self):
+        batch_size_real_data = int(math.floor(self.ratio * self.batch_size))
+        batch_size_fake_data = self.batch_size - batch_size_real_data
+
+        self.processed_idx = copy.deepcopy(self.real_data_idx)
+        rand_real_data_idx = np.random.permutation(len(self.real_data_idx) // 2)
+        for i in range(self.__len__()):
+            batch = []
+            idx_fake_data = random.sample(self.fake_data_idx, batch_size_fake_data)
+
+            for j in range(batch_size_real_data // 2):
+                idx = rand_real_data_idx[(i * batch_size_real_data + j) % (self.real_data_num // 2)]
+                batch.append(self.processed_idx[2 * idx])
+                batch.append(self.processed_idx[2 * idx + 1])
+
+            for idx in idx_fake_data:
+                batch.append(idx + self.real_data_num)
+                # batch.append(2 * idx + 1 + self.real_data_num)
+            yield batch

DSDG/DUM/Load_OULUNPUcrop_valtest.py

@@ -0,0 +1,119 @@
+from __future__ import print_function, division
+import os
+import torch
+import pandas as pd
+# from skimage import io, transform
+import cv2
+import numpy as np
+import random
+import torch
+from torch.utils.data import Dataset, DataLoader
+from torchvision import transforms
+import pdb
+import math
+import os
+
+
+class Normaliztion_valtest(object):
+    """
+        same as mxnet, normalize into [-1, 1]
+        image = (image - 127.5)/128
+    """
+
+    def __call__(self, sample):
+        image_x, val_map_x, spoofing_label, image_names = sample['image_x'], sample['val_map_x'], sample[
+            'spoofing_label'], sample['image_names']
+        new_image_x = (image_x - 127.5) / 128  # [-1,1]
+        return {'image_x': new_image_x, 'val_map_x': val_map_x, 'spoofing_label': spoofing_label,
+                'image_names': image_names}
+
+
+class ToTensor_valtest(object):
+    """
+        Convert ndarrays in sample to Tensors.
+        process only one batch every time
+    """
+
+    def __call__(self, sample):
+        image_x, val_map_x, spoofing_label, image_names = sample['image_x'], sample['val_map_x'], sample[
+            'spoofing_label'], sample['image_names']
+
+        # swap color axis because    BGR2RGB
+        # numpy image: (batch_size) x T x H x W x C
+        # torch image: (batch_size) x T x C X H X W
+        image_x = image_x[:, :, :, ::-1].transpose((0, 3, 1, 2))
+        image_x = np.array(image_x)
+
+        val_map_x = np.array(val_map_x)
+
+        spoofing_label_np = np.array([0], dtype=np.long)
+        spoofing_label_np[0] = spoofing_label
+
+        return {'image_x': torch.from_numpy(image_x.astype(np.float)).float(),
+                'val_map_x': torch.from_numpy(val_map_x.astype(np.float)).float(),
+                'spoofing_label': torch.from_numpy(spoofing_label_np.astype(np.long)).long(),
+                'image_names': image_names}
+
+
+class Spoofing_valtest(Dataset):
+
+    def __init__(self, info_list, root_dir, val_map_dir, transform=None):
+
+        self.landmarks_frame = pd.read_csv(info_list, delimiter=',', header=None)
+        self.root_dir = root_dir
+        self.val_map_dir = val_map_dir
+        self.transform = transform
+
+    def __len__(self):
+        return len(self.landmarks_frame)
+
+    def __getitem__(self, idx):
+        # print(self.landmarks_frame.iloc[idx, 0])
+        videoname = str(self.landmarks_frame.iloc[idx, 1])
+        image_path = os.path.join(self.root_dir, videoname)
+        val_map_path = os.path.join(self.val_map_dir, videoname)
+
+        image_x, val_map_x, image_names = self.get_single_image_x(image_path, val_map_path, videoname)
+
+        spoofing_label = self.landmarks_frame.iloc[idx, 0]
+        if spoofing_label == 1:
+            spoofing_label = 1  # real
+        else:
+            spoofing_label = 0
+
+        sample = {'image_x': image_x, 'val_map_x': val_map_x, 'spoofing_label': spoofing_label,
+                  'image_names': image_names}
+
+        if self.transform:
+            sample = self.transform(sample)
+        return sample
+
+    def get_single_image_x(self, images_path, maps_path, videoname):
+        # some vedio flod miss .dat
+        files_total = len([name for name in os.listdir(images_path) if os.path.isfile(os.path.join(images_path, name))])
+
+        image_x = np.zeros((files_total, 256, 256, 3))
+        map_x = np.ones((files_total, 32, 32))
+        image_names = []
+
+        file_list = os.listdir(maps_path)
+
+        for i, map_name in enumerate(file_list):
+            image_name = map_name[:-12] + '_scene.jpg'
+            image_path = os.path.join(images_path, image_name)
+            map_path = os.path.join(maps_path, map_name)
+
+            # RGB
+            image = cv2.imread(image_path)
+            # gray-map
+            map = cv2.imread(map_path, 0)
+
+            image_x[i, :, :, :] = cv2.resize(image, (256, 256))
+            # transform to binary mask --> threshold = 0 
+            map_x[i, :, :] = cv2.resize(map, (32, 32))
+            # np.where(temp < 1, temp, 1)
+            # val_map_x[i, :, :] = temp
+
+            image_names.append(image_name)
+
+        return image_x, map_x, image_names

DSDG/DUM/make_dataset/crop_dataset.py

@@ -0,0 +1,116 @@
+import math
+import os
+import cv2
+import numpy as np
+
+
+root_dir = '/export2/home/wht/oulu_images_crop/'
+
+img_root = '/export2/home/wht/oulu_images/train_img_flod/'
+map_root = '/export2/home/wht/oulu_images/train_depth_flod/'
+bbox_root = '/export2/home/wht/oulu_images/train_bbox_flod/'
+
+
+def crop_face_from_scene(image, face_name_full, scale):
+    f = open(face_name_full, 'r')
+    lines = f.readlines()
+    lines = lines[0].split(' ')
+    y1, x1, w, h = [int(ele) for ele in lines[:4]]
+    f.close()
+    y2 = y1 + w
+    x2 = x1 + h
+
+    y_mid = (y1 + y2) / 2.0
+    x_mid = (x1 + x2) / 2.0
+    h_img, w_img = image.shape[0], image.shape[1]
+    # w_img,h_img=image.size
+    w_scale = scale * w
+    h_scale = scale * h
+    y1 = y_mid - w_scale / 2.0
+    x1 = x_mid - h_scale / 2.0
+    y2 = y_mid + w_scale / 2.0
+    x2 = x_mid + h_scale / 2.0
+    y1 = max(math.floor(y1), 0)
+    x1 = max(math.floor(x1), 0)
+    y2 = min(math.floor(y2), w_img)
+    x2 = min(math.floor(x2), h_img)
+
+    # region=image[y1:y2,x1:x2]
+    region = image[x1:x2, y1:y2]
+    return region
+
+
+def crop_face_from_scene_prnet(image, face_name_full, scale):
+    h_img, w_img = image.shape[0], image.shape[1]
+    f = open(face_name_full, 'r')
+    lines = f.readlines()
+    lines = lines[0].split(' ')
+    l, r, t, b = [int(ele) for ele in lines[:4]]
+    if l < 0:
+        l = 0
+    if r > w_img:
+        r = w_img
+    if t < 0:
+        t = 0
+    if b > h_img:
+        b = h_img
+    y1 = l
+    x1 = t
+    w = r - l
+    h = b - t
+    f.close()
+    y2 = y1 + w
+    x2 = x1 + h
+
+    y_mid = (y1 + y2) / 2.0
+    x_mid = (x1 + x2) / 2.0
+    # w_img,h_img=image.size
+    w_scale = scale * w
+    h_scale = scale * h
+    y1 = y_mid - w_scale / 2.0
+    x1 = x_mid - h_scale / 2.0
+    y2 = y_mid + w_scale / 2.0
+    x2 = x_mid + h_scale / 2.0
+    y1 = max(math.floor(y1), 0)
+    x1 = max(math.floor(x1), 0)
+    y2 = min(math.floor(y2), w_img)
+    x2 = min(math.floor(x2), h_img)
+
+    # region=image[y1:y2,x1:x2]
+    region = image[x1:x2, y1:y2]
+    return region
+
+
+vedio_list = os.listdir(bbox_root)
+for i, vedio_name in enumerate(vedio_list):
+    print(i)
+    bbox_list = os.listdir(os.path.join(bbox_root, vedio_name))
+    for bbox_name in bbox_list:
+
+        face_scale = np.random.randint(12, 15)
+        face_scale = face_scale / 10.0
+
+        # face_scale = 1.3
+
+        bbox_path = os.path.join(bbox_root, vedio_name, bbox_name)
+
+        img_path = os.path.join(img_root, vedio_name, bbox_name[:-4] + '.jpg')
+
+        img = cv2.imread(img_path)
+        img_crop = cv2.resize(crop_face_from_scene_prnet(img, bbox_path, face_scale), (256, 256))
+
+        img_crop_path = os.path.join(root_dir, 'train_img_flod')
+
+        if not os.path.exists(os.path.join(img_crop_path, vedio_name)):
+            os.makedirs(os.path.join(img_crop_path, vedio_name))
+
+        cv2.imwrite(os.path.join(img_crop_path, vedio_name, bbox_name[:-4] + '.jpg'), img_crop)
+
+        map_path = os.path.join(map_root, vedio_name, bbox_name[:-9] + 'depth1D.jpg')
+
+        map = cv2.imread(map_path, 0)
+        map_crop = cv2.resize(crop_face_from_scene_prnet(map, bbox_path, face_scale), (32, 32))
+        map_crop_path = os.path.join(root_dir, 'train_depth_flod')
+        if not os.path.exists(os.path.join(map_crop_path, vedio_name)):
+            os.makedirs(os.path.join(map_crop_path, vedio_name))
+        cv2.imwrite(os.path.join(map_crop_path, vedio_name, bbox_name[:-9] + 'depth1D.jpg'), map_crop)

DSDG/DUM/models/CDCNs_u.py

@@ -0,0 +1,266 @@
+import math
+
+import torch
+import torch.nn.functional as F
+import torch.utils.model_zoo as model_zoo
+from torch import nn
+from torch.nn import Parameter
+import pdb
+import numpy as np
+
+
+class Conv2d_cd(nn.Module):
+    def __init__(self, in_channels, out_channels, kernel_size=3, stride=1,
+                 padding=1, dilation=1, groups=1, bias=False, theta=0.7):
+
+        super(Conv2d_cd, self).__init__()
+        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=padding,
+                              dilation=dilation, groups=groups, bias=bias)
+        self.theta = theta
+
+    def forward(self, x):
+        out_normal = self.conv(x)
+
+        if math.fabs(self.theta - 0.0) < 1e-8:
+            return out_normal
+        else:
+            # pdb.set_trace()
+            [C_out, C_in, kernel_size, kernel_size] = self.conv.weight.shape
+            kernel_diff = self.conv.weight.sum(2).sum(2)
+            kernel_diff = kernel_diff[:, :, None, None]
+            out_diff = F.conv2d(input=x, weight=kernel_diff, bias=self.conv.bias, stride=self.conv.stride, padding=0,
+                                groups=self.conv.groups)
+
+            return out_normal - self.theta * out_diff
+
+
+class SpatialAttention(nn.Module):
+    def __init__(self, kernel=3):
+        super(SpatialAttention, self).__init__()
+
+        self.conv1 = nn.Conv2d(2, 1, kernel_size=kernel, padding=kernel // 2, bias=False)
+        self.sigmoid = nn.Sigmoid()
+
+    def forward(self, x):
+        avg_out = torch.mean(x, dim=1, keepdim=True)
+        max_out, _ = torch.max(x, dim=1, keepdim=True)
+        x = torch.cat([avg_out, max_out], dim=1)
+        x = self.conv1(x)
+
+        return self.sigmoid(x)
+
+
+class CDCN_u(nn.Module):
+
+    def __init__(self, basic_conv=Conv2d_cd, theta=0.7):
+        super(CDCN_u, self).__init__()
+
+        self.conv1 = nn.Sequential(
+            basic_conv(3, 64, kernel_size=3, stride=1, padding=1, bias=False, theta=theta),
+            nn.BatchNorm2d(64),
+            nn.ReLU(),
+        )
+
+        self.Block1 = nn.Sequential(
+            basic_conv(64, 128, kernel_size=3, stride=1, padding=1, bias=False, theta=theta),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            basic_conv(128, 196, kernel_size=3, stride=1, padding=1, bias=False, theta=theta),
+            nn.BatchNorm2d(196),
+            nn.ReLU(),
+            basic_conv(196, 128, kernel_size=3, stride=1, padding=1, bias=False, theta=theta),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            nn.MaxPool2d(kernel_size=3, stride=2, padding=1),
+
+        )
+
+        self.Block2 = nn.Sequential(
+            basic_conv(128, 128, kernel_size=3, stride=1, padding=1, bias=False, theta=theta),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            basic_conv(128, 196, kernel_size=3, stride=1, padding=1, bias=False, theta=theta),
+            nn.BatchNorm2d(196),
+            nn.ReLU(),
+            basic_conv(196, 128, kernel_size=3, stride=1, padding=1, bias=False, theta=theta),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            nn.MaxPool2d(kernel_size=3, stride=2, padding=1),
+        )
+
+        self.Block3 = nn.Sequential(
+            basic_conv(128, 128, kernel_size=3, stride=1, padding=1, bias=False, theta=theta),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            basic_conv(128, 196, kernel_size=3, stride=1, padding=1, bias=False, theta=theta),
+            nn.BatchNorm2d(196),
+            nn.ReLU(),
+            basic_conv(196, 128, kernel_size=3, stride=1, padding=1, bias=False, theta=theta),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            nn.MaxPool2d(kernel_size=3, stride=2, padding=1),
+        )
+
+        self.lastconv1 = nn.Sequential(
+            basic_conv(128 * 3, 128, kernel_size=3, stride=1, padding=1, bias=False, theta=theta),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+        )
+
+        self.lastconv2 = nn.Sequential(
+            basic_conv(128, 64, kernel_size=3, stride=1, padding=1, bias=False, theta=theta),
+            nn.BatchNorm2d(64),
+            nn.ReLU(),
+        )
+
+        self.mu_head = nn.Sequential(
+            nn.Conv2d(64, 1, kernel_size=3, stride=1, padding=1, bias=False),
+        )
+
+        self.logvar_head = nn.Sequential(
+            nn.Conv2d(64, 1, kernel_size=3, stride=1, padding=1, bias=False),
+        )
+
+        self.downsample32x32 = nn.Upsample(size=(32, 32), mode='bilinear')
+
+    def _reparameterize(self, mu, logvar):
+        std = torch.exp(logvar).sqrt()
+        epsilon = torch.randn_like(std)
+        return mu + epsilon * std
+
+    def forward(self, x):  # x [3, 256, 256]
+
+        x_input = x
+        x = self.conv1(x)
+
+        x_Block1 = self.Block1(x)  # x [128, 128, 128]
+        x_Block1_32x32 = self.downsample32x32(x_Block1)  # x [128, 32, 32]
+
+        x_Block2 = self.Block2(x_Block1)  # x [128, 64, 64]
+        x_Block2_32x32 = self.downsample32x32(x_Block2)  # x [128, 32, 32]
+
+        x_Block3 = self.Block3(x_Block2)  # x [128, 32, 32]
+        x_Block3_32x32 = self.downsample32x32(x_Block3)  # x [128, 32, 32]
+
+        x_concat = torch.cat((x_Block1_32x32, x_Block2_32x32, x_Block3_32x32), dim=1)  # x [128*3, 32, 32]
+
+        # pdb.set_trace()
+
+        x = self.lastconv1(x_concat)  # x [128, 32, 32]
+        x = self.lastconv2(x)  # x [64, 32, 32]
+
+        mu = self.mu_head(x)
+        mu = mu.squeeze(1)
+        logvar = self.logvar_head(x)
+        logvar = logvar.squeeze(1)
+        embedding = self._reparameterize(mu, logvar)
+
+        return mu, logvar, embedding, x_concat, x_Block1, x_Block2, x_Block3, x_input
+
+
+class depthnet_u(nn.Module):
+
+    def __init__(self):
+        super(depthnet_u, self).__init__()
+
+        self.conv1 = nn.Sequential(
+            nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(64),
+            nn.ReLU(),
+        )
+
+        self.Block1 = nn.Sequential(
+            nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            nn.Conv2d(128, 196, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(196),
+            nn.ReLU(),
+            nn.Conv2d(196, 128, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            nn.MaxPool2d(kernel_size=3, stride=2, padding=1),
+
+        )
+
+        self.Block2 = nn.Sequential(
+            nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            nn.Conv2d(128, 196, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(196),
+            nn.ReLU(),
+            nn.Conv2d(196, 128, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            nn.MaxPool2d(kernel_size=3, stride=2, padding=1),
+        )
+
+        self.Block3 = nn.Sequential(
+            nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            nn.Conv2d(128, 196, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(196),
+            nn.ReLU(),
+            nn.Conv2d(196, 128, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            nn.MaxPool2d(kernel_size=3, stride=2, padding=1),
+        )
+
+        self.lastconv1 = nn.Sequential(
+            nn.Conv2d(128 * 3, 128, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+        )
+
+        self.lastconv2 = nn.Sequential(
+            nn.Conv2d(128, 64, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(64),
+            nn.ReLU(),
+        )
+
+        self.mu_head = nn.Sequential(
+            nn.Conv2d(64, 1, kernel_size=3, stride=1, padding=1, bias=False),
+        )
+
+        self.logvar_head = nn.Sequential(
+            nn.Conv2d(64, 1, kernel_size=3, stride=1, padding=1, bias=False),
+        )
+
+        self.downsample32x32 = nn.Upsample(size=(32, 32), mode='bilinear')
+
+    def _reparameterize(self, mu, logvar):
+        std = torch.exp(logvar).sqrt()
+        epsilon = torch.randn_like(std)
+        return mu + epsilon * std
+
+    def forward(self, x):  # x [3, 256, 256]
+
+        x_input = x
+        x = self.conv1(x)
+
+        x_Block1 = self.Block1(x)  # x [128, 128, 128]
+        x_Block1_32x32 = self.downsample32x32(x_Block1)  # x [128, 32, 32]
+
+        x_Block2 = self.Block2(x_Block1)  # x [128, 64, 64]
+        x_Block2_32x32 = self.downsample32x32(x_Block2)  # x [128, 32, 32]
+
+        x_Block3 = self.Block3(x_Block2)  # x [128, 32, 32]
+        x_Block3_32x32 = self.downsample32x32(x_Block3)  # x [128, 32, 32]
+
+        x_concat = torch.cat((x_Block1_32x32, x_Block2_32x32, x_Block3_32x32), dim=1)  # x [128*3, 32, 32]
+
+        # pdb.set_trace()
+
+        x = self.lastconv1(x_concat)  # x [128, 32, 32]
+        x = self.lastconv2(x)  # x [64, 32, 32]
+
+        mu = self.mu_head(x)
+        mu = mu.squeeze(1)
+        logvar = self.logvar_head(x)
+        logvar = logvar.squeeze(1)
+        embedding = self._reparameterize(mu, logvar)
+
+        return mu, logvar, embedding, x_concat, x_Block1, x_Block2, x_Block3, x_input

DSDG/DUM/models/ResNet_u.py

@@ -0,0 +1,114 @@
+import math
+
+import torch
+import torch.nn.functional as F
+import torch.utils.model_zoo as model_zoo
+from torch import nn
+from torch.nn import Parameter
+import pdb
+import numpy as np
+
+
+class ResidualBlock(nn.Module):
+    def __init__(self, inchannel, outchannel, stride=1):
+        super(ResidualBlock, self).__init__()
+        self.left = nn.Sequential(
+            nn.Conv2d(inchannel, outchannel, kernel_size=3, stride=stride, padding=1, bias=False),
+            nn.BatchNorm2d(outchannel),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(outchannel, outchannel, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(outchannel)
+        )
+        self.shortcut = nn.Sequential()
+        if stride != 1 or inchannel != outchannel:
+            self.shortcut = nn.Sequential(
+                nn.Conv2d(inchannel, outchannel, kernel_size=1, stride=stride, bias=False),
+                nn.BatchNorm2d(outchannel)
+            )
+
+    def forward(self, x):
+        out = self.left(x)
+        out += self.shortcut(x)
+        out = F.relu(out)
+        return out
+
+
+class ResNet(nn.Module):
+    def __init__(self, ResidualBlock = ResidualBlock):
+        super(ResNet, self).__init__()
+        self.inchannel = 64
+        self.conv1 = nn.Sequential(
+            nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(64),
+            nn.ReLU(),
+        )
+        self.layer1 = self.make_layer(ResidualBlock, 64,  2, stride=1)
+        self.layer2 = self.make_layer(ResidualBlock, 128, 2, stride=2)
+        self.layer3 = self.make_layer(ResidualBlock, 196, 2, stride=2)
+        self.layer4 = self.make_layer(ResidualBlock, 256, 2, stride=2)
+        self.lastconv1 = nn.Sequential(
+            nn.Conv2d(128 + 196 + 256, 128, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+        )
+        self.lastconv2 = nn.Sequential(
+            nn.Conv2d(128, 64, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.BatchNorm2d(64),
+            nn.ReLU(),
+        )
+        self.lastconv3 = nn.Sequential(
+            nn.Conv2d(64, 1, kernel_size=3, stride=1, padding=1, bias=False),
+            nn.ReLU(),
+        )
+        self.downsample32x32 = nn.Upsample(size=(32, 32), mode='bilinear')
+        
+        self.mu_head = nn.Sequential(
+            nn.Conv2d(64, 1, kernel_size=3, stride=1, padding=1, bias=False),
+            
+        )
+        self.logvar_head = nn.Sequential(
+            nn.Conv2d(64, 1, kernel_size=3, stride=1, padding=1, bias=False),
+            
+        )
+
+    def make_layer(self, block, channels, num_blocks, stride):
+        strides = [stride] + [1] * (num_blocks - 1)   #strides=[1,1]
+        layers = []
+        for stride in strides:
+            layers.append(block(self.inchannel, channels, stride))
+            self.inchannel = channels
+        return nn.Sequential(*layers)
+
+    def _reparameterize(self, mu, logvar):
+        std = torch.exp(logvar).sqrt()
+        epsilon = torch.randn_like(std)
+        return mu + epsilon * std
+
+    def forward(self, x):
+        x_input = x
+        x = self.conv1(x)
+        x_block1 = self.layer1(x)
+        x_block2 = self.layer2(x_block1)
+        x_block2_32 = self.downsample32x32(x_block2)
+        x_block3 = self.layer3(x_block2)
+        x_block3_32 = self.downsample32x32(x_block3)
+        x_block4 = self.layer4(x_block3)
+        x_block4_32 = self.downsample32x32(x_block4)
+
+        x_concat = torch.cat((x_block2_32, x_block3_32, x_block4_32), dim=1)
+
+        x = self.lastconv1(x_concat)
+        x = self.lastconv2(x)
+        mu = self.mu_head(x)
+        mu = mu.squeeze(1)
+        logvar = self.logvar_head(x)
+        logvar = logvar.squeeze(1)
+        embedding = self._reparameterize(mu, logvar)
+
+        return mu, logvar, embedding, x_concat, x_block2, x_block3, x_block4, x_input
+
+
+def ResNet18_u():
+
+    return ResNet(ResidualBlock)
+

DSDG/DUM/test.py

@@ -0,0 +1,168 @@
+from __future__ import print_function, division
+import torch
+
+torch.set_printoptions(profile="full")
+import matplotlib.pyplot as plt
+import argparse, os
+import numpy as np
+import shutil
+from torch.utils.data import DataLoader
+from torchvision import transforms
+
+from models.CDCNs_u import Conv2d_cd, CDCN_u
+
+from Load_OULUNPUcrop_valtest import Spoofing_valtest, Normaliztion_valtest, ToTensor_valtest
+
+import torch.optim as optim
+
+from utils import performances
+
+# Dataset root
+
+val_image_dir = '/export2/home/wht/oulu_img_crop/dev_file_flod/'
+test_image_dir = '/export2/home/wht/oulu_img_crop/test_file_flod/'
+
+val_map_dir = '/export2/home/wht/oulu_img_crop/dev_depth_flod/'
+test_map_dir = '/export2/home/wht/oulu_img_crop/test_depth_flod/'
+
+val_list = '/export2/home/wht/oulu_img_crop/protocols/Protocol_1/Dev.txt'
+test_list = '/export2/home/wht/oulu_img_crop/protocols/Protocol_1/Test.txt'
+
+
+# main function
+def test():
+    # GPU  & log file  -->   if use DataParallel, please comment this command
+    os.environ["CUDA_VISIBLE_DEVICES"] = '0, 1, 2, 3'
+
+    isExists = os.path.exists(args.log)
+    if not isExists:
+        os.makedirs(args.log)
+    log_file = open(args.log + '/' + args.log + '_log_P1.txt', 'w')
+
+    log_file.write('Oulu-NPU, P1:\n ')
+    log_file.flush()
+
+    print('test!\n')
+    log_file.write('test!\n')
+    log_file.flush()
+
+    model = CDCN_u(basic_conv=Conv2d_cd, theta=0.7)
+    # model = ResNet18_u()
+
+    model = model.cuda()
+    model = torch.nn.DataParallel(model)
+    model.load_state_dict(torch.load('./DUM/checkpoint/CDCN_U_P1.pkl', map_location='cuda:0'))
+
+    print(model)
+
+    optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=0.00005)
+
+    for epoch in range(args.epochs):
+
+        model.eval()
+
+        with torch.no_grad():
+            ###########################################
+            '''                val             '''
+            ###########################################
+            # val for threshold
+            val_data = Spoofing_valtest(val_list, val_image_dir, val_map_dir,
+                                        transform=transforms.Compose([Normaliztion_valtest(), ToTensor_valtest()]))
+            dataloader_val = DataLoader(val_data, batch_size=1, shuffle=False, num_workers=4)
+
+            map_score_list = []
+
+            for i, sample_batched in enumerate(dataloader_val):
+                # get the inputs
+                inputs, spoof_label = sample_batched['image_x'].cuda(), sample_batched['spoofing_label'].cuda()
+                val_maps = sample_batched['val_map_x'].cuda()  # binary map from PRNet
+
+                optimizer.zero_grad()
+
+                # pdb.set_trace()
+                map_score = 0.0
+                for frame_t in range(inputs.shape[1]):
+                    mu, logvar, map_x, x_concat, x_Block1, x_Block2, x_Block3, x_input = model(
+                        inputs[:, frame_t, :, :, :])
+                    score_norm = torch.sum(mu) / torch.sum(val_maps[:, frame_t, :, :])
+                    map_score += score_norm
+
+                map_score = map_score / inputs.shape[1]
+
+                map_score_list.append('{} {}\n'.format(map_score, spoof_label[0][0]))
+                # pdb.set_trace()
+            map_score_val_filename = args.log + '/' + args.log + '_map_score_val.txt'
+            with open(map_score_val_filename, 'w') as file:
+                file.writelines(map_score_list)
+
+            ###########################################
+            '''                test             '''
+            ##########################################
+            # test for ACC
+            test_data = Spoofing_valtest(test_list, test_image_dir, test_map_dir,
+                                         transform=transforms.Compose([Normaliztion_valtest(), ToTensor_valtest()]))
+            dataloader_test = DataLoader(test_data, batch_size=1, shuffle=False, num_workers=4)
+
+            map_score_list = []
+
+            for i, sample_batched in enumerate(dataloader_test):
+                # get the inputs
+                inputs, spoof_label = sample_batched['image_x'].cuda(), sample_batched['spoofing_label'].cuda()
+                test_maps = sample_batched['val_map_x'].cuda()
+
+                optimizer.zero_grad()
+
+                # pdb.set_trace()
+                map_score = 0.0
+                for frame_t in range(inputs.shape[1]):
+                    mu, logvar, map_x, x_concat, x_Block1, x_Block2, x_Block3, x_input = model(
+                        inputs[:, frame_t, :, :, :])
+
+                    score_norm = torch.sum(mu) / torch.sum(test_maps[:, frame_t, :, :])
+                    map_score += score_norm
+
+                map_score = map_score / inputs.shape[1]
+
+                map_score_list.append('{} {}\n'.format(map_score, spoof_label[0][0]))
+
+            map_score_test_filename = args.log + '/' + args.log + '_map_score_test.txt'
+            with open(map_score_test_filename, 'w') as file:
+                file.writelines(map_score_list)
+
+            #############################################################
+            #       performance measurement both val and test
+            #############################################################
+            val_threshold, test_threshold, val_ACC, val_ACER, test_ACC, test_APCER, test_BPCER, test_ACER, test_ACER_test_threshold = performances(
+                map_score_val_filename, map_score_test_filename)
+
+            print('epoch:%d, Val:  val_threshold= %.4f, val_ACC= %.4f, val_ACER= %.4f' % (
+                epoch + 1, val_threshold, val_ACC, val_ACER))
+            log_file.write('\n epoch:%d, Val:  val_threshold= %.4f, val_ACC= %.4f, val_ACER= %.4f \n' % (
+                epoch + 1, val_threshold, val_ACC, val_ACER))
+
+            print('epoch:%d, Test:  ACC= %.4f, APCER= %.4f, BPCER= %.4f, ACER= %.4f' % (
+                epoch + 1, test_ACC, test_APCER, test_BPCER, test_ACER))
+            log_file.write('epoch:%d, Test:  ACC= %.4f, APCER= %.4f, BPCER= %.4f, ACER= %.4f \n' % (
+                epoch + 1, test_ACC, test_APCER, test_BPCER, test_ACER))
+            log_file.flush()
+
+    print('Finished Training')
+    log_file.close()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="save quality using landmarkpose model")
+    parser.add_argument('--gpus', type=str, default='0,1,2,3', help='the gpu id used for predict')
+    parser.add_argument('--lr', type=float, default=0.001, help='initial learning rate')
+    parser.add_argument('--batchsize', type=int, default=32, help='initial batchsize')
+    parser.add_argument('--step_size', type=int, default=500, help='how many epochs lr decays once')  # 500
+    parser.add_argument('--gamma', type=float, default=0.5, help='gamma of optim.lr_scheduler.StepLR, decay of lr')
+    parser.add_argument('--kl_lambda', type=float, default=0.001, help='')
+    parser.add_argument('--echo_batches', type=int, default=50, help='how many batches display once')  # 50
+    parser.add_argument('--epochs', type=int, default=1, help='total training epochs')
+    parser.add_argument('--log', type=str, default="CDCN_U_P1_test", help='log and save model name')
+    parser.add_argument('--finetune', action='store_true', default=False, help='whether finetune other models')
+    parser.add_argument('--test', action='store_true', default=True, help='')
+
+    args = parser.parse_args()
+    test()

DSDG/DUM/train.py

@@ -0,0 +1,289 @@
+from __future__ import print_function, division
+import torch
+import matplotlib.pyplot as plt
+import argparse, os
+import numpy as np
+from torch.utils.data import DataLoader
+from torchvision import transforms
+
+from models.CDCNs_u import Conv2d_cd, CDCN_u
+
+from Load_OULUNPUcrop_train import Spoofing_train_g, SeparateBatchSampler, Normaliztion, ToTensor, \
+    RandomHorizontalFlip, Cutout, RandomErasing
+from Load_OULUNPUcrop_valtest import Spoofing_valtest, Normaliztion_valtest, ToTensor_valtest
+
+import torch.nn.functional as F
+import torch.nn as nn
+import torch.optim as optim
+
+from utils import AvgrageMeter, performances
+
+# Dataset root
+train_image_dir = '/export2/home/wht/oulu_img_crop/train_file_flod/'
+val_image_dir = '/export2/home/wht/oulu_img_crop/dev_file_flod/'
+test_image_dir = '/export2/home/wht/oulu_img_crop/test_file_flod/'
+
+train_map_dir = '/export2/home/wht/oulu_img_crop/train_depth_flod/'
+val_map_dir = '/export2/home/wht/oulu_img_crop/dev_depth_flod/'
+test_map_dir = '/export2/home/wht/oulu_img_crop/test_depth_flod/'
+
+train_list = '/export2/home/wht/oulu_img_crop/protocols/Protocol_1/Train_g.txt'
+val_list = '/export2/home/wht/oulu_img_crop/protocols/Protocol_1/Dev.txt'
+test_list = '/export2/home/wht/oulu_img_crop/protocols/Protocol_1/Test.txt'
+
+
+def contrast_depth_conv(input):
+    ''' compute contrast depth in both of (out, label) '''
+    '''
+        input  32x32
+        output 8x32x32
+    '''
+
+    kernel_filter_list = [
+        [[1, 0, 0], [0, -1, 0], [0, 0, 0]], [[0, 1, 0], [0, -1, 0], [0, 0, 0]], [[0, 0, 1], [0, -1, 0], [0, 0, 0]],
+        [[0, 0, 0], [1, -1, 0], [0, 0, 0]], [[0, 0, 0], [0, -1, 1], [0, 0, 0]],
+        [[0, 0, 0], [0, -1, 0], [1, 0, 0]], [[0, 0, 0], [0, -1, 0], [0, 1, 0]], [[0, 0, 0], [0, -1, 0], [0, 0, 1]]
+    ]
+
+    kernel_filter = np.array(kernel_filter_list, np.float32)
+
+    kernel_filter = torch.from_numpy(kernel_filter.astype(np.float)).float().cuda()
+    # weights (in_channel, out_channel, kernel, kernel)
+    kernel_filter = kernel_filter.unsqueeze(dim=1)
+
+    input = input.unsqueeze(dim=1).expand(input.shape[0], 8, input.shape[1], input.shape[2])
+
+    contrast_depth = F.conv2d(input, weight=kernel_filter, groups=8)
+
+    return contrast_depth
+
+
+class Contrast_depth_loss(nn.Module):
+    def __init__(self):
+        super(Contrast_depth_loss, self).__init__()
+        return
+
+    def forward(self, out, label):
+        contrast_out = contrast_depth_conv(out)
+        contrast_label = contrast_depth_conv(label)
+
+        criterion_MSE = nn.MSELoss().cuda()
+
+        loss = criterion_MSE(contrast_out, contrast_label)
+
+        return loss
+
+
+def train_test():
+    isExists = os.path.exists(args.log)
+    if not isExists:
+        os.makedirs(args.log)
+    log_file = open(args.log + '/' + args.log + '_log_P1.txt', 'a')
+
+    log_file.write('Oulu-NPU, P1:\n ')
+    log_file.flush()
+
+    print('train from scratch!\n')
+    log_file.write('train from scratch!\n')
+    log_file.write('lr:%.6f, lamda_kl:%.6f , batchsize:%d\n' % (args.lr, args.kl_lambda, args.batchsize))
+    log_file.flush()
+
+    model = CDCN_u(basic_conv=Conv2d_cd, theta=0.7)
+    # model = ResNet18_u()
+
+    model = model.cuda()
+    model = torch.nn.DataParallel(model)
+
+    lr = args.lr
+    optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=0.00005)
+    scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=args.step_size, gamma=args.gamma)
+
+    print(model)
+
+    criterion_absolute_loss = nn.MSELoss().cuda()
+    criterion_contrastive_loss = Contrast_depth_loss().cuda()
+
+    for epoch in range(args.epochs):
+        if (epoch + 1) % args.step_size == 0:
+            lr *= args.gamma
+
+        loss_absolute_real = AvgrageMeter()
+        loss_absolute_fake = AvgrageMeter()
+        loss_contra_real = AvgrageMeter()
+        loss_contra_fake = AvgrageMeter()
+        loss_kl_real = AvgrageMeter()
+        loss_kl_fake = AvgrageMeter()
+
+        ###########################################
+        '''                train             '''
+        ###########################################
+        model.train()
+
+        # load random 16-frame clip data every epoch
+        train_data = Spoofing_train_g(train_list, train_image_dir, train_map_dir,
+                                      transform=transforms.Compose(
+                                          [RandomErasing(), RandomHorizontalFlip(), ToTensor(), Cutout(),
+                                           Normaliztion()]))
+        train_real_idx, train_fake_idx = train_data.get_idx()
+        batch_sampler = SeparateBatchSampler(train_real_idx, train_fake_idx, batch_size=args.batchsize, ratio=args.ratio)
+        dataloader_train = DataLoader(train_data, num_workers=8, batch_sampler=batch_sampler)
+
+        for i, sample_batched in enumerate(dataloader_train):
+            # get the inputs
+            inputs, map_label, spoof_label = sample_batched['image_x'].cuda(), sample_batched['map_x'].cuda(), \
+                                             sample_batched['spoofing_label'].cuda()
+
+            optimizer.zero_grad()
+
+            # forward + backward + optimize
+            mu, logvar, map_x, x_concat, x_Block1, x_Block2, x_Block3, x_input = model(inputs)
+
+            mu_real = mu[:int(args.batchsize * args.ratio), :, :]
+            logvar_real = logvar[:int(args.batchsize * args.ratio), :, :]
+            map_x_real = map_x[:int(args.batchsize * args.ratio), :, :]
+            map_label_real = map_label[:int(args.batchsize * args.ratio), :, :]
+
+            absolute_loss_real = criterion_absolute_loss(map_x_real, map_label_real)
+            contrastive_loss_real = criterion_contrastive_loss(map_x_real, map_label_real)
+            kl_loss_real = -(1 + logvar_real - (mu_real - map_label_real).pow(2) - logvar_real.exp()) / 2
+            kl_loss_real = kl_loss_real.sum(dim=1).sum(dim=1).mean()
+            kl_loss_real = args.kl_lambda * kl_loss_real
+
+            mu_fake = mu[int(args.batchsize * args.ratio):, :, :]
+            logvar_fake = logvar[int(args.batchsize * args.ratio):, :, :]
+            map_x_fake = map_x[int(args.batchsize * args.ratio):, :, :]
+            map_label_fake = map_label[int(args.batchsize * args.ratio):, :, :]
+
+            absolute_loss_fake = 0.1 * criterion_absolute_loss(map_x_fake, map_label_fake)
+            contrastive_loss_fake = 0.1 * criterion_contrastive_loss(map_x_fake, map_label_fake)
+            kl_loss_fake = -(1 + logvar_fake - (mu_fake - map_label_fake).pow(2) - logvar_fake.exp()) / 2
+            kl_loss_fake = kl_loss_fake.sum(dim=1).sum(dim=1).mean()
+            kl_loss_fake = 0.1 * args.kl_lambda * kl_loss_fake
+
+            absolute_loss = absolute_loss_real + absolute_loss_fake
+            contrastive_loss = contrastive_loss_real + contrastive_loss_fake
+            kl_loss = kl_loss_real + kl_loss_fake
+
+            loss = absolute_loss + contrastive_loss + kl_loss
+
+            loss.backward()
+
+            optimizer.step()
+
+            n = inputs.size(0)
+            loss_absolute_real.update(absolute_loss_real.data, n)
+            loss_absolute_fake.update(absolute_loss_fake.data, n)
+            loss_contra_real.update(contrastive_loss_real.data, n)
+            loss_contra_fake.update(contrastive_loss_fake.data, n)
+            loss_kl_real.update(kl_loss_real.data, n)
+            loss_kl_fake.update(kl_loss_fake.data, n)
+
+        scheduler.step()
+        # whole epoch average
+        print(
+            'epoch:%d, Train:  Absolute_loss: real=%.4f,fake=%.4f, '
+            'Contrastive_loss: real=%.4f,fake=%.4f, kl_loss: real=%.4f,fake=%.4f' % (
+                epoch + 1, loss_absolute_real.avg, loss_absolute_fake.avg, loss_contra_real.avg, loss_contra_fake.avg,
+                loss_kl_real.avg, loss_kl_fake.avg))
+
+        # validation/test
+        if epoch < 200:
+            epoch_test = 200
+        else:
+            epoch_test = 50
+        # epoch_test = 1
+        if epoch % epoch_test == epoch_test - 1:
+            model.eval()
+
+            with torch.no_grad():
+                ###########################################
+                '''                val             '''
+                ###########################################
+                # val for threshold
+                val_data = Spoofing_valtest(val_list, val_image_dir, val_map_dir,
+                                            transform=transforms.Compose([Normaliztion_valtest(), ToTensor_valtest()]))
+                dataloader_val = DataLoader(val_data, batch_size=1, shuffle=False, num_workers=4)
+
+                map_score_list = []
+
+                for i, sample_batched in enumerate(dataloader_val):
+                    # get the inputs
+                    inputs, spoof_label = sample_batched['image_x'].cuda(), sample_batched['spoofing_label'].cuda()
+                    val_maps = sample_batched['val_map_x'].cuda()  # binary map from PRNet
+
+                    optimizer.zero_grad()
+
+                    mu, logvar, map_x, x_concat, x_Block1, x_Block2, x_Block3, x_input = model(inputs.squeeze(0))
+                    score_norm = mu.sum(dim=1).sum(dim=1) / val_maps.squeeze(0).sum(dim=1).sum(dim=1)
+                    map_score = score_norm.mean()
+                    map_score_list.append('{} {}\n'.format(map_score, spoof_label[0][0]))
+
+                map_score_val_filename = args.log + '/' + args.log + '_map_score_val.txt'
+                with open(map_score_val_filename, 'w') as file:
+                    file.writelines(map_score_list)
+
+                ###########################################
+                '''                test             '''
+                ##########################################
+                # test for ACC
+                test_data = Spoofing_valtest(test_list, test_image_dir, test_map_dir,
+                                             transform=transforms.Compose([Normaliztion_valtest(), ToTensor_valtest()]))
+                dataloader_test = DataLoader(test_data, batch_size=1, shuffle=False, num_workers=4)
+
+                map_score_list = []
+
+                for i, sample_batched in enumerate(dataloader_test):
+                    # get the inputs
+                    inputs, spoof_label = sample_batched['image_x'].cuda(), sample_batched['spoofing_label'].cuda()
+                    test_maps = sample_batched['val_map_x'].cuda()
+
+                    optimizer.zero_grad()
+                    mu, logvar, map_x, x_concat, x_Block1, x_Block2, x_Block3, x_input = model(inputs.squeeze(0))
+                    score_norm = mu.sum(dim=1).sum(dim=1) / test_maps.squeeze(0).sum(dim=1).sum(dim=1)
+                    map_score = score_norm.mean()
+                    map_score_list.append('{} {}\n'.format(map_score, spoof_label[0][0]))
+
+                map_score_test_filename = args.log + '/' + args.log + '_map_score_test.txt'
+                with open(map_score_test_filename, 'w') as file:
+                    file.writelines(map_score_list)
+
+                #############################################################
+                #       performance measurement both val and test
+                #############################################################
+                val_threshold, test_threshold, val_ACC, val_ACER, test_ACC, test_APCER, test_BPCER, test_ACER, test_ACER_test_threshold = performances(
+                    map_score_val_filename, map_score_test_filename)
+
+                print('epoch:%d, Val:  val_threshold= %.4f, val_ACC= %.4f, val_ACER= %.4f' % (
+                    epoch + 1, val_threshold, val_ACC, val_ACER))
+                log_file.write('\n epoch:%d, Val:  val_threshold= %.4f, val_ACC= %.4f, val_ACER= %.4f \n' % (
+                    epoch + 1, val_threshold, val_ACC, val_ACER))
+
+                print('epoch:%d, Test:  ACC= %.4f, APCER= %.4f, BPCER= %.4f, ACER= %.4f' % (
+                    epoch + 1, test_ACC, test_APCER, test_BPCER, test_ACER))
+                log_file.write('epoch:%d, Test:  ACC= %.4f, APCER= %.4f, BPCER= %.4f, ACER= %.4f \n' % (
+                    epoch + 1, test_ACC, test_APCER, test_BPCER, test_ACER))
+                log_file.flush()
+
+        if epoch % epoch_test == epoch_test - 1:
+            # save the model until the next improvement
+            torch.save(model.state_dict(), args.log + '/' + args.log + '_%d.pkl' % (epoch + 1))
+
+    print('Finished Training')
+    log_file.close()
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="save quality using landmarkpose model")
+    parser.add_argument('--gpus', type=str, default='0, 1, 2, 3', help='the gpu id used for predict')
+    parser.add_argument('--lr', type=float, default=0.0001, help='initial learning rate')
+    parser.add_argument('--batchsize', type=int, default=64, help='initial batchsize')
+    parser.add_argument('--step_size', type=int, default=500, help='how many epochs lr decays once')  # 500
+    parser.add_argument('--gamma', type=float, default=0.5, help='gamma of optim.lr_scheduler.StepLR, decay of lr')
+    parser.add_argument('--kl_lambda', type=float, default=0.001, help='')
+    parser.add_argument('--ratio', type=float, default=0.75, help='real and fake in batchsize ')
+    parser.add_argument('--echo_batches', type=int, default=50, help='how many batches display once')  # 50
+    parser.add_argument('--epochs', type=int, default=1600, help='total training epochs')
+    parser.add_argument('--log', type=str, default="CDCN_U_P1", help='log and save model name')
+    parser.add_argument('--finetune', action='store_true', default=False, help='whether finetune other models')
+
+    args = parser.parse_args()
+    train_test()

DSDG/DUM/utils.py

@@ -0,0 +1,431 @@
+import os
+import numpy as np
+import torch
+import shutil
+import torchvision.transforms as transforms
+from torch.autograd import Variable
+import sklearn
+from sklearn import metrics
+from sklearn.metrics import roc_curve, auc
+import pdb
+
+
+class AvgrageMeter(object):
+
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.avg = 0
+        self.sum = 0
+        self.cnt = 0
+
+    def update(self, val, n=1):
+        self.sum += val * n
+        self.cnt += n
+        self.avg = self.sum / self.cnt
+
+
+def accuracy(output, target, topk=(1,)):
+    maxk = max(topk)
+    batch_size = target.size(0)
+
+    _, pred = output.topk(maxk, 1, True, True)
+    pred = pred.t()
+    correct = pred.eq(target.view(1, -1).expand_as(pred))
+
+    res = []
+    for k in topk:
+        correct_k = correct[:k].view(-1).float().sum(0)
+        res.append(correct_k.mul_(100.0 / batch_size))
+    return res
+
+
+def get_threshold(score_file):
+    with open(score_file, 'r') as file:
+        lines = file.readlines()
+
+    data = []
+    count = 0.0
+    num_real = 0.0
+    num_fake = 0.0
+    for line in lines:
+        count += 1
+        tokens = line.split()
+        angle = float(tokens[0])
+        # pdb.set_trace()
+        type = int(tokens[1])
+        data.append({'map_score': angle, 'label': type})
+        if type == 1:
+            num_real += 1
+        else:
+            num_fake += 1
+
+    min_error = count  # account ACER (or ACC)
+    min_threshold = 0.0
+    min_ACC = 0.0
+    min_ACER = 0.0
+    min_APCER = 0.0
+    min_BPCER = 0.0
+
+    for d in data:
+        threshold = d['map_score']
+
+        type1 = len([s for s in data if s['map_score'] <= threshold and s['label'] == 1])
+        type2 = len([s for s in data if s['map_score'] > threshold and s['label'] == 0])
+
+        ACC = 1 - (type1 + type2) / count
+        APCER = type2 / num_fake
+        BPCER = type1 / num_real
+        ACER = (APCER + BPCER) / 2.0
+
+        if ACER < min_error:
+            min_error = ACER
+            min_threshold = threshold
+            min_ACC = ACC
+            min_ACER = ACER
+            min_APCER = APCER
+            min_BPCER = min_BPCER
+
+    # print(min_error, min_threshold)
+    return min_threshold, min_ACC, min_APCER, min_BPCER, min_ACER
+
+
+def test_threshold_based(threshold, score_file):
+    with open(score_file, 'r') as file:
+        lines = file.readlines()
+
+    data = []
+    count = 0.0
+    num_real = 0.0
+    num_fake = 0.0
+    for line in lines:
+        count += 1
+        tokens = line.split()
+        angle = float(tokens[0])
+        type = int(tokens[1])
+        data.append({'map_score': angle, 'label': type})
+        if type == 1:
+            num_real += 1
+        else:
+            num_fake += 1
+
+    type1 = len([s for s in data if s['map_score'] <= threshold and s['label'] == 1])
+    type2 = len([s for s in data if s['map_score'] > threshold and s['label'] == 0])
+
+    ACC = 1 - (type1 + type2) / count
+    APCER = type2 / num_fake
+    BPCER = type1 / num_real
+    ACER = (APCER + BPCER) / 2.0
+
+    return ACC, APCER, BPCER, ACER
+
+
+def get_err_threhold(fpr, tpr, threshold):
+    RightIndex = (tpr + (1 - fpr) - 1)
+    right_index = np.argmax(RightIndex)
+    best_th = threshold[right_index]
+    err = fpr[right_index]
+
+    differ_tpr_fpr_1 = tpr + fpr - 1.0
+
+    right_index = np.argmin(np.abs(differ_tpr_fpr_1))
+    best_th = threshold[right_index]
+    err = fpr[right_index]
+
+    # print(err, best_th)
+    return err, best_th
+
+
+# def performances(dev_scores, dev_labels, test_scores, test_labels):
+def performances(map_score_val_filename, map_score_test_filename):
+    # val
+    with open(map_score_val_filename, 'r') as file:
+        lines = file.readlines()
+    val_scores = []
+    val_labels = []
+    data = []
+    count = 0.0
+    num_real = 0.0
+    num_fake = 0.0
+    for line in lines:
+        count += 1
+        tokens = line.split()
+        score = float(tokens[0])
+        label = float(tokens[1])  # label = int(tokens[1])
+        val_scores.append(score)
+        val_labels.append(label)
+        data.append({'map_score': score, 'label': label})
+        if label == 1:
+            num_real += 1
+        else:
+            num_fake += 1
+
+    fpr, tpr, threshold = roc_curve(val_labels, val_scores, pos_label=1)
+    val_err, val_threshold = get_err_threhold(fpr, tpr, threshold)
+
+    type1 = len([s for s in data if s['map_score'] <= val_threshold and s['label'] == 1])
+    type2 = len([s for s in data if s['map_score'] > val_threshold and s['label'] == 0])
+
+    val_ACC = 1 - (type1 + type2) / count
+    val_APCER = type2 / num_fake
+    val_BPCER = type1 / num_real
+    val_ACER = (val_APCER + val_BPCER) / 2.0
+
+    # test 
+    with open(map_score_test_filename, 'r') as file2:
+        lines = file2.readlines()
+    test_scores = []
+    test_labels = []
+    data = []
+    count = 0.0
+    num_real = 0.0
+    num_fake = 0.0
+    for line in lines:
+        count += 1
+        tokens = line.split()
+        score = float(tokens[0])
+        label = float(tokens[1])  # label = int(tokens[1])
+        test_scores.append(score)
+        test_labels.append(label)
+        data.append({'map_score': score, 'label': label})
+        if label == 1:
+            num_real += 1
+        else:
+            num_fake += 1
+
+    # test based on val_threshold     
+    type1 = len([s for s in data if s['map_score'] <= val_threshold and s['label'] == 1])
+    print([s for s in data if s['map_score'] <= val_threshold and s['label'] == 1])
+    type2 = len([s for s in data if s['map_score'] > val_threshold and s['label'] == 0])
+    print([s for s in data if s['map_score'] > val_threshold and s['label'] == 0])
+
+    test_ACC = 1 - (type1 + type2) / count
+    test_APCER = type2 / num_fake
+    test_BPCER = type1 / num_real
+    test_ACER = (test_APCER + test_BPCER) / 2.0
+
+    # test based on test_threshold     
+    fpr_test, tpr_test, threshold_test = roc_curve(test_labels, test_scores, pos_label=1)
+    err_test, best_test_threshold = get_err_threhold(fpr_test, tpr_test, threshold_test)
+
+    type1 = len([s for s in data if s['map_score'] <= best_test_threshold and s['label'] == 1])
+    type2 = len([s for s in data if s['map_score'] > best_test_threshold and s['label'] == 0])
+
+    test_threshold_ACC = 1 - (type1 + type2) / count
+    test_threshold_APCER = type2 / num_fake
+    test_threshold_BPCER = type1 / num_real
+    test_threshold_ACER = (test_threshold_APCER + test_threshold_BPCER) / 2.0
+
+    return val_threshold, best_test_threshold, val_ACC, val_ACER, test_ACC, test_APCER, test_BPCER, test_ACER, test_threshold_ACER
+
+
+def performances_SiW_EER(map_score_val_filename):
+    # val
+    with open(map_score_val_filename, 'r') as file:
+        lines = file.readlines()
+    val_scores = []
+    val_labels = []
+    data = []
+    count = 0.0
+    num_real = 0.0
+    num_fake = 0.0
+    for line in lines:
+        count += 1
+        tokens = line.split()
+        score = float(tokens[0])
+        label = int(tokens[1])
+        val_scores.append(score)
+        val_labels.append(label)
+        data.append({'map_score': score, 'label': label})
+        if label == 1:
+            num_real += 1
+        else:
+            num_fake += 1
+
+    fpr, tpr, threshold = roc_curve(val_labels, val_scores, pos_label=1)
+    val_err, val_threshold = get_err_threhold(fpr, tpr, threshold)
+
+    type1 = len([s for s in data if s['map_score'] <= val_threshold and s['label'] == 1])
+    type2 = len([s for s in data if s['map_score'] > val_threshold and s['label'] == 0])
+
+    val_ACC = 1 - (type1 + type2) / count
+    val_APCER = type2 / num_fake
+    val_BPCER = type1 / num_real
+    val_ACER = (val_APCER + val_BPCER) / 2.0
+
+    return val_threshold, val_ACC, val_APCER, val_BPCER, val_ACER
+
+
+def performances_SiWM_EER(map_score_val_filename):
+    # val
+    with open(map_score_val_filename, 'r') as file:
+        lines = file.readlines()
+    val_scores = []
+    val_labels = []
+    data = []
+    count = 0.0
+    num_real = 0.0
+    num_fake = 0.0
+    for line in lines:
+        count += 1
+        tokens = line.split()
+        score = float(tokens[0])
+        label = int(tokens[1])
+        val_scores.append(score)
+        val_labels.append(label)
+        data.append({'map_score': score, 'label': label})
+        if label == 1:
+            num_real += 1
+        else:
+            num_fake += 1
+
+    fpr, tpr, threshold = roc_curve(val_labels, val_scores, pos_label=1)
+    val_err, val_threshold = get_err_threhold(fpr, tpr, threshold)
+
+    type1 = len([s for s in data if s['map_score'] <= val_threshold and s['label'] == 1])
+    type2 = len([s for s in data if s['map_score'] > val_threshold and s['label'] == 0])
+
+    val_ACC = 1 - (type1 + type2) / count
+    val_APCER = type2 / num_fake
+    val_BPCER = type1 / num_real
+    val_ACER = (val_APCER + val_BPCER) / 2.0
+
+    return val_threshold, val_err, val_ACC, val_APCER, val_BPCER, val_ACER
+
+
+def get_err_threhold_CASIA_Replay(fpr, tpr, threshold):
+    RightIndex = (tpr + (1 - fpr) - 1)
+    right_index = np.argmax(RightIndex)
+    best_th = threshold[right_index]
+    err = fpr[right_index]
+
+    differ_tpr_fpr_1 = tpr + fpr - 1.0
+
+    right_index = np.argmin(np.abs(differ_tpr_fpr_1))
+    best_th = threshold[right_index]
+    err = fpr[right_index]
+
+    # print(err, best_th)
+    return err, best_th, right_index
+
+
+def performances_CASIA_Replay(map_score_val_filename):
+    # val
+    with open(map_score_val_filename, 'r') as file:
+        lines = file.readlines()
+    val_scores = []
+    val_labels = []
+    data = []
+    count = 0.0
+    num_real = 0.0
+    num_fake = 0.0
+    for line in lines:
+        count += 1
+        tokens = line.split()
+        score = float(tokens[0])
+        label = float(tokens[1])  # int(tokens[1])
+        val_scores.append(score)
+        val_labels.append(label)
+        data.append({'map_score': score, 'label': label})
+        if label == 1:
+            num_real += 1
+        else:
+            num_fake += 1
+
+    fpr, tpr, threshold = roc_curve(val_labels, val_scores, pos_label=1)
+    val_err, val_threshold, right_index = get_err_threhold_CASIA_Replay(fpr, tpr, threshold)
+
+    type1 = len([s for s in data if s['map_score'] <= val_threshold and s['label'] == 1])
+    print([s for s in data if s['map_score'] <= val_threshold and s['label'] == 1])
+    type2 = len([s for s in data if s['map_score'] > val_threshold and s['label'] == 0])
+    print([s for s in data if s['map_score'] > val_threshold and s['label'] == 0])
+
+    val_ACC = 1 - (type1 + type2) / count
+
+    FRR = 1 - tpr  # FRR = 1 - TPR
+
+    HTER = (fpr + FRR) / 2.0  # error recognition rate &  reject recognition rate
+
+    return val_ACC, fpr[right_index], FRR[right_index], HTER[right_index], val_threshold
+
+
+def performances_ZeroShot(map_score_val_filename):
+    # val
+    with open(map_score_val_filename, 'r') as file:
+        lines = file.readlines()
+    val_scores = []
+    val_labels = []
+    data = []
+    count = 0.0
+    num_real = 0.0
+    num_fake = 0.0
+    for line in lines:
+        count += 1
+        tokens = line.split()
+        score = float(tokens[0])
+        label = int(tokens[1])
+        val_scores.append(score)
+        val_labels.append(label)
+        data.append({'map_score': score, 'label': label})
+        if label == 1:
+            num_real += 1
+        else:
+            num_fake += 1
+
+    fpr, tpr, threshold = roc_curve(val_labels, val_scores, pos_label=1)
+    auc_val = metrics.auc(fpr, tpr)
+
+    val_err, val_threshold, right_index = get_err_threhold_CASIA_Replay(fpr, tpr, threshold)
+
+    type1 = len([s for s in data if s['map_score'] <= val_threshold and s['label'] == 1])
+    type2 = len([s for s in data if s['map_score'] > val_threshold and s['label'] == 0])
+
+    val_ACC = 1 - (type1 + type2) / count
+
+    FRR = 1 - tpr  # FRR = 1 - TPR
+
+    HTER = (fpr + FRR) / 2.0  # error recognition rate &  reject recognition rate
+
+    return val_ACC, auc_val, HTER[right_index]
+
+
+def count_parameters_in_MB(model):
+    return np.sum(np.prod(v.size()) for name, v in model.named_parameters() if "auxiliary" not in name) / 1e6
+
+
+def save_checkpoint(state, is_best, save):
+    filename = os.path.join(save, 'checkpoint.pth.tar')
+    torch.save(state, filename)
+    if is_best:
+        best_filename = os.path.join(save, 'model_best.pth.tar')
+        shutil.copyfile(filename, best_filename)
+
+
+def save(model, model_path):
+    torch.save(model.state_dict(), model_path)
+
+
+def load(model, model_path):
+    model.load_state_dict(torch.load(model_path))
+
+
+def drop_path(x, drop_prob):
+    if drop_prob > 0.:
+        keep_prob = 1. - drop_prob
+        mask = Variable(torch.cuda.FloatTensor(x.size(0), 1, 1, 1, 1).bernoulli_(keep_prob))
+        x.div_(keep_prob)
+        x.mul_(mask)
+    return x
+
+
+def create_exp_dir(path, scripts_to_save=None):
+    if not os.path.exists(path):
+        os.mkdir(path)
+    print('Experiment dir : {}'.format(path))
+
+    if scripts_to_save is not None:
+        os.mkdir(os.path.join(path, 'scripts'))
+    for script in scripts_to_save:
+        dst_file = os.path.join(path, 'scripts', os.path.basename(script))
+        shutil.copyfile(script, dst_file)

DSDG/misc/__init__.py

@@ -0,0 +1,2 @@
+
+from .util import *

DSDG/misc/util.py

@@ -0,0 +1,109 @@
+import os
+import time
+import argparse
+import numpy as np
+
+import torch
+import torch.nn.functional as F
+
+
+def rgb2gray(img):
+    r, g, b = torch.split(img, 1, dim=1)
+    return torch.mul(r, 0.299) + torch.mul(g, 0.587) + torch.mul(b, 0.114)
+
+
+def reparameterize(mu, logvar):
+    std = logvar.mul(0.5).exp_()
+    eps = torch.cuda.FloatTensor(std.size()).normal_()
+    return eps.mul(std).add_(mu)
+
+
+def kl_loss(mu, logvar, prior_mu=0):
+    v_kl = mu.add(-prior_mu).pow(2).add_(logvar.exp()).mul_(-1).add_(1).add_(logvar)
+    v_kl = v_kl.sum(dim=-1).mul_(-0.5)  # (batch, 2)
+    return v_kl
+
+
+def reconstruction_loss(prediction, target, size_average=False):
+    error = (prediction - target).view(prediction.size(0), -1)
+    error = error ** 2
+    error = torch.sum(error, dim=-1)
+
+    if size_average:
+        error = error.mean()
+    else:
+        error = error.sum()
+    return error
+
+
+def load_model(model, pretrained):
+    weights = torch.load(pretrained)
+    pretrained_dict = weights['model'].state_dict()
+    model_dict = model.state_dict()
+    # 1. filter out unnecessary keys
+    pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}
+    # 2. overwrite entries in the existing state dict
+    model_dict.update(pretrained_dict)
+    # 3. load the new state dict
+    model.load_state_dict(model_dict)
+
+
+def save_checkpoint(model_path, model, epoch, iteration, name):
+    model_out_path = model_path + name + "model_epoch_{}_iter_{}.pth".format(epoch, iteration)
+    state = {"epoch": epoch, "model": model}
+    if not os.path.exists(model_path):
+        os.makedirs(model_path)
+    torch.save(state, model_out_path)
+    print("Checkpoint saved to {}".format(model_out_path))
+
+
+class AverageMeter(object):
+    """Computes and stores the average and current value"""
+
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+
+    def update(self, val, n=1):
+        self.val = val
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+
+
+def MMD_Loss(fc_nir, fc_vis):
+    mean_fc_nir = torch.mean(fc_nir, 0)
+    mean_fc_vis = torch.mean(fc_vis, 0)
+    loss_mmd = F.mse_loss(mean_fc_nir, mean_fc_vis)
+    return loss_mmd
+
+
+def adjust_learning_rate(lr, step, optimizer, epoch):
+    scale = 0.457305051927326
+    lr = lr * (scale ** (epoch // step))
+    print('lr: {}'.format(lr))
+    if (epoch != 0) & (epoch % step == 0):
+        print('Change lr')
+        for param_group in optimizer.param_groups:
+            param_group['lr'] = param_group['lr'] * scale
+
+
+def accuracy(output, target, topk=(1,)):
+    """Computes the precision@k for the specified values of k"""
+    maxk = max(topk)
+    batch_size = target.size(0)
+
+    _, pred = output.topk(maxk, 1, True, True)
+    pred = pred.t()
+    correct = pred.eq(target.view(1, -1).expand_as(pred))
+
+    res = []
+    for k in topk:
+        correct_k = correct[:k].view(-1).float().sum(0)
+        res.append(correct_k.mul_(100.0 / batch_size))
+    return res

DSDG/networks/__init__.py

@@ -0,0 +1,32 @@
+import torch
+from .generator import Encoder, Decoder, Encoder_s, Decoder_s, Cls
+from .light_cnn import network_29layers_v2, resblock
+
+
+# define generator
+def define_G(hdim=256, attack_type=4):
+    netE_nir = Encoder_s(hdim=hdim)
+    netCls = Cls(hdim=hdim, attack_type=attack_type)
+    netE_vis = Encoder(hdim=hdim)
+    netG = Decoder_s(hdim=hdim)
+
+    netE_nir = torch.nn.DataParallel(netE_nir).cuda()
+    netE_vis = torch.nn.DataParallel(netE_vis).cuda()
+    netG = torch.nn.DataParallel(netG).cuda()
+    netCls = torch.nn.DataParallel(netCls).cuda()
+
+    return netE_nir, netE_vis, netG, netCls
+
+
+# define identity preserving && feature extraction net
+def define_IP(is_train=False):
+    netIP = network_29layers_v2(resblock, [1, 2, 3, 4], is_train)
+    netIP = torch.nn.DataParallel(netIP).cuda()
+    return netIP
+
+
+# define recognition network
+def LightCNN_29v2(num_classes=10000, is_train=True):
+    net = network_29layers_v2(resblock, [1, 2, 3, 4], is_train, num_classes=num_classes)
+    net = torch.nn.DataParallel(net).cuda()
+    return net

DSDG/networks/generator.py

@@ -0,0 +1,192 @@
+import numpy as np
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+
+
+class Encoder(nn.Module):
+    def __init__(self, hdim=256):
+        super(Encoder, self).__init__()
+
+        self.hdim = hdim
+
+        self.main = nn.Sequential(
+            nn.Conv2d(3, 32, 5, 2, 2, bias=False),
+            nn.InstanceNorm2d(32, eps=0.001),
+            nn.LeakyReLU(0.2),
+            nn.AvgPool2d(2),
+            make_layer(_Residual_Block, 1, 32, 64),
+            nn.AvgPool2d(2),
+            make_layer(_Residual_Block, 1, 64, 128),
+            nn.AvgPool2d(2),
+            make_layer(_Residual_Block, 1, 128, 256),
+            nn.AvgPool2d(2),
+            make_layer(_Residual_Block, 1, 256, 512),
+            nn.AvgPool2d(2),
+            make_layer(_Residual_Block, 1, 512, 512)
+        )
+
+        # mu and logvar
+        self.fc = nn.Linear(512 * 4 * 4, 2 * hdim)
+
+    def forward(self, x):
+        z = self.main(x).view(x.size(0), -1)
+        z = self.fc(z)
+        mu, logvar = torch.split(z, split_size_or_sections=self.hdim, dim=-1)
+
+        return mu, logvar
+
+
+class Encoder_s(nn.Module):
+    def __init__(self, hdim=256):
+        super(Encoder_s, self).__init__()
+
+        self.hdim = hdim
+
+        self.main = nn.Sequential(
+            nn.Conv2d(3, 32, 5, 2, 2, bias=False),
+            nn.InstanceNorm2d(32, eps=0.001),
+            nn.LeakyReLU(0.2),
+            nn.AvgPool2d(2),
+            make_layer(_Residual_Block, 1, 32, 64),
+            nn.AvgPool2d(2),
+            make_layer(_Residual_Block, 1, 64, 128),
+            nn.AvgPool2d(2),
+            make_layer(_Residual_Block, 1, 128, 256),
+            nn.AvgPool2d(2),
+            make_layer(_Residual_Block, 1, 256, 512),
+            nn.AvgPool2d(2),
+            make_layer(_Residual_Block, 1, 512, 512)
+        )
+
+        # mu and logvar
+        self.fc = nn.Linear(512 * 4 * 4, 4 * hdim)
+        # self.fc_at = nn.Linear(hdim, attack_type)
+
+    def forward(self, x):
+        z = self.main(x).view(x.size(0), -1)
+        z = self.fc(z)
+        mu, logvar, mu_a, logvar_a = torch.split(z, split_size_or_sections=self.hdim, dim=-1)
+        # a_type = self.fc_at(a_t)
+
+        return mu, logvar, mu_a, logvar_a
+
+
+class Cls(nn.Module):
+    def __init__(self, hdim=256, attack_type=4):
+        super(Cls, self).__init__()
+
+        self.fc = nn.Linear(hdim, attack_type)
+
+    def forward(self, x):
+        a_cls = self.fc(x)
+
+        return a_cls
+
+class Decoder(nn.Module):
+    def __init__(self, hdim=256):
+        super(Decoder, self).__init__()
+
+        self.fc = nn.Sequential(
+            nn.Linear(hdim, 512 * 4 * 4),
+            nn.ReLU(True)
+        )
+
+        self.main = nn.Sequential(
+            make_layer(_Residual_Block, 1, 512, 512),
+            nn.Upsample(scale_factor=2, mode='nearest'),
+            make_layer(_Residual_Block, 1, 512, 512),
+            nn.Upsample(scale_factor=2, mode='nearest'),
+            make_layer(_Residual_Block, 1, 512, 512),
+            nn.Upsample(scale_factor=2, mode='nearest'),
+            make_layer(_Residual_Block, 1, 512, 256),
+            nn.Upsample(scale_factor=2, mode='nearest'),
+            make_layer(_Residual_Block, 1, 256, 128),
+            nn.Upsample(scale_factor=2, mode='nearest'),
+            make_layer(_Residual_Block, 2, 128, 64),
+            nn.Upsample(scale_factor=2, mode='nearest'),
+            nn.Conv2d(64, 3 + 3, 5, 1, 2)
+        )
+
+    def forward(self, z):
+        z = z.view(z.size(0), -1)
+        y = self.fc(z)
+        x = y.view(z.size(0), -1, 4, 4)
+        img = torch.sigmoid(self.main(x))
+        return img
+
+
+class Decoder_s(nn.Module):
+    def __init__(self, hdim=256):
+        super(Decoder_s, self).__init__()
+
+        self.fc = nn.Sequential(
+            nn.Linear(3 * hdim, 512 * 4 * 4),
+            nn.ReLU(True)
+        )
+
+        self.main = nn.Sequential(
+            make_layer(_Residual_Block, 1, 512, 512),
+            nn.Upsample(scale_factor=2, mode='nearest'),
+            make_layer(_Residual_Block, 1, 512, 512),
+            nn.Upsample(scale_factor=2, mode='nearest'),
+            make_layer(_Residual_Block, 1, 512, 512),
+            nn.Upsample(scale_factor=2, mode='nearest'),
+            make_layer(_Residual_Block, 1, 512, 256),
+            nn.Upsample(scale_factor=2, mode='nearest'),
+            make_layer(_Residual_Block, 1, 256, 128),
+            nn.Upsample(scale_factor=2, mode='nearest'),
+            make_layer(_Residual_Block, 2, 128, 64),
+            nn.Upsample(scale_factor=2, mode='nearest'),
+            nn.Conv2d(64, 3 + 3, 5, 1, 2)
+        )
+
+    def forward(self, z):
+        z = z.view(z.size(0), -1)
+        y = self.fc(z)
+        x = y.view(z.size(0), -1, 4, 4)
+        img = torch.sigmoid(self.main(x))
+        return img
+
+
+class _Residual_Block(nn.Module):
+    def __init__(self, inc=64, outc=64, groups=1):
+        super(_Residual_Block, self).__init__()
+
+        if inc is not outc:
+            self.conv_expand = nn.Conv2d(in_channels=inc, out_channels=outc, kernel_size=1, stride=1, padding=0,
+                                         groups=1, bias=False)
+        else:
+            self.conv_expand = None
+
+        self.conv1 = nn.Conv2d(in_channels=inc, out_channels=outc, kernel_size=3, stride=1, padding=1, groups=groups,
+                               bias=False)
+        self.bn1 = nn.InstanceNorm2d(outc, eps=0.001)
+        self.relu1 = nn.LeakyReLU(0.2, inplace=True)
+        self.conv2 = nn.Conv2d(in_channels=outc, out_channels=outc, kernel_size=3, stride=1, padding=1, groups=groups,
+                               bias=False)
+        self.bn2 = nn.InstanceNorm2d(outc, eps=0.001)
+        self.relu2 = nn.LeakyReLU(0.2, inplace=True)
+
+    def forward(self, x):
+        if self.conv_expand is not None:
+            identity_data = self.conv_expand(x)
+        else:
+            identity_data = x
+
+        output = self.relu1(self.bn1(self.conv1(x)))
+        output = self.conv2(output)
+        output = self.relu2(self.bn2(torch.add(output, identity_data)))
+        return output
+
+
+def make_layer(block, num_of_layer, inc=64, outc=64, groups=1):
+    if num_of_layer < 1:
+        num_of_layer = 1
+    layers = []
+    layers.append(block(inc=inc, outc=outc, groups=groups))
+    for _ in range(1, num_of_layer):
+        layers.append(block(inc=outc, outc=outc, groups=groups))
+    return nn.Sequential(*layers)

DSDG/networks/light_cnn.py

@@ -0,0 +1,98 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class mfm(nn.Module):
+    def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=1, type=1):
+        super(mfm, self).__init__()
+        self.out_channels = out_channels
+        if type == 1:
+            self.filter = nn.Conv2d(in_channels, 2 * out_channels, kernel_size=kernel_size, stride=stride,
+                                    padding=padding)
+        else:
+            self.filter = nn.Linear(in_channels, 2 * out_channels)
+
+    def forward(self, x):
+        x = self.filter(x)
+        out = torch.split(x, self.out_channels, 1)
+        return torch.max(out[0], out[1])
+
+
+class group(nn.Module):
+    def __init__(self, in_channels, out_channels, kernel_size, stride, padding):
+        super(group, self).__init__()
+        self.conv_a = mfm(in_channels, in_channels, 1, 1, 0)
+        self.conv = mfm(in_channels, out_channels, kernel_size, stride, padding)
+
+    def forward(self, x):
+        x = self.conv_a(x)
+        x = self.conv(x)
+        return x
+
+
+class resblock(nn.Module):
+    def __init__(self, in_channels, out_channels):
+        super(resblock, self).__init__()
+        self.conv1 = mfm(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+        self.conv2 = mfm(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+
+    def forward(self, x):
+        res = x
+        out = self.conv1(x)
+        out = self.conv2(out)
+        out = out + res
+        return out
+
+
+class network_29layers_v2(nn.Module):
+    def __init__(self, block, layers, is_train=False, num_classes=80013):
+        super(network_29layers_v2, self).__init__()
+        self.is_train = is_train
+        self.conv1 = mfm(1, 48, 5, 1, 2)
+        self.block1 = self._make_layer(block, layers[0], 48, 48)
+        self.group1 = group(48, 96, 3, 1, 1)
+        self.block2 = self._make_layer(block, layers[1], 96, 96)
+        self.group2 = group(96, 192, 3, 1, 1)
+        self.block3 = self._make_layer(block, layers[2], 192, 192)
+        self.group3 = group(192, 128, 3, 1, 1)
+        self.block4 = self._make_layer(block, layers[3], 128, 128)
+        self.group4 = group(128, 128, 3, 1, 1)
+        self.fc = nn.Linear(8 * 8 * 128, 256)
+
+        if self.is_train:
+            self.fc2_ = nn.Linear(256, num_classes, bias=False)
+
+    def _make_layer(self, block, num_blocks, in_channels, out_channels):
+        layers = []
+        for i in range(0, num_blocks):
+            layers.append(block(in_channels, out_channels))
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = F.max_pool2d(x, 2) + F.avg_pool2d(x, 2)
+
+        x = self.block1(x)
+        x = self.group1(x)
+        x = F.max_pool2d(x, 2) + F.avg_pool2d(x, 2)
+
+        x = self.block2(x)
+        x = self.group2(x)
+        x = F.max_pool2d(x, 2) + F.avg_pool2d(x, 2)
+
+        x = self.block3(x)
+        x = self.group3(x)
+        x = self.block4(x)
+        x = self.group4(x)
+        x = F.max_pool2d(x, 2) + F.avg_pool2d(x, 2)
+
+        x = x.view(x.size(0), -1)
+        fc = self.fc(x)
+
+        if self.is_train:
+            x = F.dropout(fc, training=self.training)
+            out = self.fc2_(x)
+            return out, fc
+        else:
+            return fc

app.py

@@ -1,3 +1,6 @@
+import subprocess
+subprocess.run(["sh", "tddfa/build.sh"]) 
+
 import gradio as gr
 from gradio.components import Dropdown
 
@@ -12,20 +15,25 @@ import pandas as pd
 from skimage.io import imread, imsave
 # from tddfa.TDDFA import TDDFA
 from tddfa.utils.depth import depth
-from tddfa.TDDFA_ONNX import TDDFA_ONNX
+from tddfa.TDDFA import TDDFA
+
+import torch.optim as optim
+from DSDG.DUM.models.CDCNs_u import Conv2d_cd, CDCN_u
 
 import os
 os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
 os.environ['OMP_NUM_THREADS'] = '4'
 
+
+device = torch.device("cpu")
 labels = ['Live', 'Spoof']
-thresh = 0.45
+pix_threshhold = 0.45
+dsdg_threshold = 0.003
 examples = [
     ['examples/1_1_21_2_33_scene_fake.jpg', "DeePixBiS"],
     ['examples/frame150_real.jpg', "DeePixBiS"],
     ['examples/1_2.avi_125_real.jpg', "DeePixBiS"],
     ['examples/1_3.avi_25_fake.jpg', "DeePixBiS"]]
-device = torch.device("cpu")
 faceClassifier = cv.CascadeClassifier('./DeePixBiS/Classifiers/haarface.xml')
 tfms = transforms.Compose([
     transforms.ToPILImage(),
@@ -37,8 +45,52 @@ model = DeePixBiS(pretrained=False)
 model.load_state_dict(torch.load('./DeePixBiS/DeePixBiS.pth'))
 model.eval()
 
-cfg = yaml.load(open('tddfa/configs/mb1_120x120.yml'), Loader=yaml.SafeLoader)
-tddfa = TDDFA_ONNX(gpu_mode=False, **cfg)
+
+depth_config_path = 'tddfa/configs/mb05_120x120.yml'  # 'tddfa/configs/mb1_120x120.yml
+cfg = yaml.load(open(depth_config_path), Loader=yaml.SafeLoader)
+tddfa = TDDFA(gpu_mode=False, **cfg)
+
+
+model = CDCN_u(basic_conv=Conv2d_cd, theta=0.7)
+model = model.to(device)
+weights = torch.load('./DSDG/DUM/checkpoint/CDCN_U_P1_updated.pkl', map_location=device)
+model.load_state_dict(weights)
+optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=0.00005)
+model.eval()
+
+
+class Normaliztion_valtest(object):
+    """
+        same as mxnet, normalize into [-1, 1]
+        image = (image - 127.5)/128
+    """
+    def __call__(self, image_x):
+        image_x = (image_x - 127.5) / 128  # [-1,1]
+        return image_x
+
+
+def prepare_data(images, boxes, depths):
+    transform = transforms.Compose([Normaliztion_valtest()])
+    files_total = 1
+    image_x = np.zeros((files_total, 256, 256, 3))
+    depth_x = np.ones((files_total, 32, 32))
+
+    for i, (image, bbox, depth_img) in enumerate(
+            zip(images, boxes, depths)):
+        x, y, w, h = bbox
+        depth_img = cv.cvtColor(depth_img, cv.COLOR_RGB2GRAY)
+        image = image[y:y + h, x:x + w]
+        depth_img = depth_img[y:y + h, x:x + w]
+
+        image_x[i, :, :, :] = cv.resize(image, (256, 256))
+        # transform to binary mask --> threshold = 0 
+        depth_x[i, :, :] = cv.resize(depth_img, (32, 32))
+    image_x = image_x.transpose((0, 3, 1, 2))
+    image_x = transform(image_x)
+    image_x = torch.from_numpy(image_x.astype(float)).float()
+    depth_x = torch.from_numpy(depth_x.astype(float)).float()
+    return image_x, depth_x
+
 
 def find_largest_face(faces):
     largest_face = None
@@ -51,6 +103,7 @@ def find_largest_face(faces):
             largest_face = (x, y, w, h)
     return largest_face
 
+
 def inference(img, model_name):
     confidences = {}
     grey = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
@@ -68,14 +121,9 @@ def inference(img, model_name):
         if model_name == 'DeePixBiS':
             mask, binary = model.forward(faceRegion)
             res = torch.mean(mask).item()
-            if res < thresh:
-                cls = 'Spoof'
-                color = (0, 0, 255)
-                res = 1 - res
-            else:
-                cls = 'Real'
-                color = (0, 255, 0)
-            
+            cls = 'Real' if res >= pix_threshhold else 'Spoof'
+            res = 1 - res
+
         else:
             dense_flag = True
             boxes = list(face)
@@ -83,13 +131,35 @@ def inference(img, model_name):
             param_lst, roi_box_lst = tddfa(img, [boxes])
             
             ver_lst = tddfa.recon_vers(param_lst, roi_box_lst, dense_flag=dense_flag)
-            img = depth(img, ver_lst, tddfa.tri, with_bg_flag=False)
-            cls = 'Other'
-            res = 0.5
-            color = (0, 0, 255)
+            depth_img = depth(img, ver_lst, tddfa.tri, with_bg_flag=False)
+            with torch.no_grad():
+                map_score_list = []
+                image_x, map_x = prepare_data([img], [list(face)], [depth_img])
+                # get the inputs
+                image_x = image_x.unsqueeze(0)
+                map_x = map_x.unsqueeze(0)
+                inputs = image_x.to(device)
+                test_maps = map_x.to(device)
+                optimizer.zero_grad()
+                
+                map_score = 0.0
+                for frame_t in range(inputs.shape[1]):
+                    mu, logvar, map_x, x_concat, x_Block1, x_Block2, x_Block3, x_input = model(inputs[:, frame_t, :, :, :])
+
+                    score_norm = torch.sum(mu) / torch.sum(test_maps[:, frame_t, :, :])
+                    map_score += score_norm
+                map_score = map_score / inputs.shape[1]
+                map_score_list.append(map_score)
+
+            res = map_score_list[0].item()
+            if res > 10:
+                res = 0.0
+            cls = 'Real' if res >= dsdg_threshold else 'Spoof'
+            res = res * 100
 
         label = f'{cls} {res:.2f}'
         confidences = {label: res}
+        color = color = (0, 255, 0) if cls == 'Real' else (255, 0, 0)
         cv.rectangle(img, (x, y), (x + w, y + h), color, 2)
         cv.putText(img, label, (x, y + h + 30),
                     cv.FONT_HERSHEY_COMPLEX, 1, color)

tddfa/TDDFA.py

@@ -10,13 +10,13 @@ import torch
 from torchvision.transforms import Compose
 import torch.backends.cudnn as cudnn
 
-import models
-from bfm import BFMModel
-from utils.io import _load
-from utils.functions import (
+import tddfa.models as models
+from tddfa.bfm import BFMModel
+from tddfa.utils.io import _load
+from tddfa.utils.functions import (
     crop_img, parse_roi_box_from_bbox, parse_roi_box_from_landmark,
 )
-from utils.tddfa_util import (
+from tddfa.utils.tddfa_util import (
     load_model, _parse_param, similar_transform,
     ToTensorGjz, NormalizeGjz
 )
@@ -114,7 +114,7 @@ class TDDFA(object):
             else:
                 param = self.model(inp)
 
-            param = param.squeeze().cpu().numpy().flatten().astype(np.float32)
+            param = param.squeeze().cpu().detach().numpy().flatten().astype(np.float32)
             param = param * self.param_std + self.param_mean  # re-scale
             # print('output', param)
             param_lst.append(param)

tddfa/weights/mb1_120x120.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91aba763fd0a5d9ed78d11ff728be54b83e24ab2f2b3389f204be95b482b23d2
+size 13048589