init

.gitattributes

@@ -29,3 +29,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text

Asm_assisted_loss.py

@@ -0,0 +1,69 @@
+import tensorflow as tf
+from pca_utility import PCAUtility
+import numpy as np
+
+
+class ASMLoss:
+    def __init__(self, dataset_name, accuracy):
+        self.dataset_name = dataset_name
+        self.accuracy = accuracy
+
+    def calculate_pose_loss(self, x_pr, x_gt):
+        return tf.reduce_mean(tf.square(x_gt - x_pr))
+
+    def calculate_landmark_ASM_assisted_loss(self, landmark_pr, landmark_gt, current_epoch, total_steps):
+        """
+        :param landmark_pr:
+        :param landmark_gt:
+        :param current_epoch:
+        :param total_steps:
+        :return:
+        """
+        # calculating ASMLoss weight:
+        asm_weight = 0.5
+        if current_epoch < total_steps//3: asm_weight = 2.0
+        elif total_steps//3 <= current_epoch < 2*total_steps//3: asm_weight = 1.0
+
+        # creating the ASM-ground truth
+        landmark_gt_asm = self._calculate_asm(input_tensor=landmark_gt)
+
+        # calculating ASMLoss
+        asm_loss = tf.reduce_mean(tf.square(landmark_gt_asm - landmark_pr))
+
+        # calculating MSELoss
+        mse_loss = tf.reduce_mean(tf.square(landmark_gt - landmark_pr))
+
+        # calculating total loss
+        return mse_loss + asm_weight * asm_loss
+
+    def _calculate_asm(self, input_tensor):
+        pca_utility = PCAUtility()
+        eigenvalues, eigenvectors, meanvector = pca_utility.load_pca_obj(self.dataset_name, pca_percentages=self.accuracy)
+
+        input_vector = np.array(input_tensor)
+        out_asm_vector = []
+        batch_size = input_vector.shape[0]
+        for i in range(batch_size):
+            b_vector_p = self._calculate_b_vector(input_vector[i], eigenvalues, eigenvectors, meanvector)
+            out_asm_vector.append(meanvector + np.dot(eigenvectors, b_vector_p))
+
+        out_asm_vector = np.array(out_asm_vector)
+        return out_asm_vector
+
+    def _calculate_b_vector(self, predicted_vector, eigenvalues, eigenvectors, meanvector):
+        b_vector = np.dot(eigenvectors.T, predicted_vector - meanvector)
+        # revised b to be in -3lambda =>
+        i = 0
+        for b_item in b_vector:
+            lambda_i_sqr = 3 * np.sqrt(eigenvalues[i])
+            if b_item > 0:
+                b_item = min(b_item, lambda_i_sqr)
+            else:
+                b_item = max(b_item, -1 * lambda_i_sqr)
+            b_vector[i] = b_item
+            i += 1
+
+        return b_vector
+
+
+

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Ali Pourramezan Fard
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,3 +1,146 @@
----
-license: mit
----
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/deep-active-shape-model-for-face-alignment/pose-estimation-on-300w-full)](https://paperswithcode.com/sota/pose-estimation-on-300w-full?p=deep-active-shape-model-for-face-alignment)
+
+	
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/deep-active-shape-model-for-face-alignment/face-alignment-on-wflw)](https://paperswithcode.com/sota/face-alignment-on-wflw?p=deep-active-shape-model-for-face-alignment)
+
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/deep-active-shape-model-for-face-alignment/face-alignment-on-300w)](https://paperswithcode.com/sota/face-alignment-on-300w?p=deep-active-shape-model-for-face-alignment)
+
+```diff
+! plaese STAR the repo if you like it.
+```
+
+# [ASMNet](https://scholar.google.com/scholar?oi=bibs&cluster=3428857185978099736&btnI=1&hl=en)
+
+
+## a Lightweight Deep Neural Network for Face Alignment and Pose Estimation
+
+#### Link to the paper:
+https://scholar.google.com/scholar?oi=bibs&cluster=3428857185978099736&btnI=1&hl=en
+
+#### Link to the paperswithcode.com:
+https://paperswithcode.com/paper/asmnet-a-lightweight-deep-neural-network-for
+
+#### Link to the article on Towardsdatascience.com:
+https://aliprf.medium.com/asmnet-a-lightweight-deep-neural-network-for-face-alignment-and-pose-estimation-9e9dfac07094
+
+```
+Please cite this work as:
+
+      @inproceedings{fard2021asmnet,
+            title={ASMNet: A Lightweight Deep Neural Network for Face Alignment and Pose Estimation},
+            author={Fard, Ali Pourramezan and Abdollahi, Hojjat and Mahoor, Mohammad},
+            booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
+            pages={1521--1530},
+            year={2021}
+      }
+```
+
+## Introduction
+
+ ASMNet is a lightweight Convolutional Neural Network (CNN) which is designed to perform face alignment and pose estimation efficiently while having acceptable accuracy. ASMNet proposed inspired by MobileNetV2, modified to be suitable for face alignment and pose
+estimation, while being about 2 times smaller in terms of number of the parameters. Moreover, Inspired by Active Shape Model (ASM), ASM-assisted loss function is proposed in order to improve the accuracy of facial landmark points detection and pose estimation.
+
+## ASMnet Architecture
+ Features in a CNN are distributed hierarchically. In other words, the lower layers have features such as edges, and corners which are more suitable for tasks like landmark localization and pose estimation, and deeper layers contain more abstract features that are more suitable for tasks like image classification and image detection. Furthermore, training a network for correlated tasks simultaneously builds a synergy that can improve the performance of each task.
+
+ Having said that, we designed ASMNe by fusing the features that are available if different layers of the model. Furthermore, by concatenating the features that are collected after each global average pooling layer in the back-propagation process, it will be possible for the network to evaluate the effect of each shortcut path. Following is the ASMNet architecture:
+
+![ASMNet architecture](https://github.com/aliprf/ASMNet/blob/master/documents/graphical_items_in_paper/arch.png?raw=true)
+
+The implementation of ASMNet in TensorFlow is provided in the following path:
+https://github.com/aliprf/ASMNet/blob/master/cnn_model.py
+
+
+
+## ASM Loss
+
+We proposed a new loss function called ASM-LOSS which utilizes ASM to improve the accuracy of the network. In other words, during the training process, the loss function compares the predicted facial landmark points with their corresponding ground truth as well as the smoothed version the ground truth which is generated using ASM operator. Accordingly, ASM-LOSS guides the network to first learn the smoothed distribution of the facial landmark points. Then, it leads the network to learn the original landmark points. For more detail please refer to the paper.
+Following is the ASM Loss diagram:
+
+![ASM Loss](https://github.com/aliprf/ASMNet/blob/master/documents/graphical_items_in_paper/Lossfunction.png?raw=true)
+
+
+## Evaluation
+
+As you can see in the following tables, ASMNet has only 1.4 M parameters which is the smallets comparing to the similar Facial landmark points detection models. Moreover, ASMNet designed to performs Face alignment as well as Pose estimation with a very small CNN while having an acceptable accuracy. 
+
+![num of params](https://github.com/aliprf/ASMNet/blob/master/documents/graphical_items_in_paper/num_params.png?raw=true)
+
+Although ASMNet is much smaller than the state-of-the-art methods on face alignment, it's performance is also very good and acceptable for many real-world applications:
+![300W Evaluation](https://github.com/aliprf/ASMNet/blob/master/documents/graphical_items_in_paper/300wEval.png?raw=true)
+
+![WFLW Evaluation](https://github.com/aliprf/ASMNet/blob/master/documents/graphical_items_in_paper/wflwEval.png?raw=true)
+
+
+As shown in the following table, ASMNet performs much better that the state-of-the-art models on 300W dataseton Pose estimation task:
+![Pose Evaluation](https://github.com/aliprf/ASMNet/blob/master/documents/graphical_items_in_paper/poseEval.png?raw=true)
+
+
+Following are some samples in order to show the visual performance of ASMNet on 300W and WFLW datasets:
+![300W visual](https://github.com/aliprf/ASMNet/blob/master/documents/graphical_items_in_paper/300W.png?raw=true)
+![wflw visual](https://github.com/aliprf/ASMNet/blob/master/documents/graphical_items_in_paper/wflw.png?raw=true)
+
+The visual performance of Pose estimation task using ASMNet is very accurate and the results also are much better than the state-of-the-art pose estimation over 300W dataset:
+
+![pose sample visual](https://github.com/aliprf/ASMNet/blob/master/documents/graphical_items_in_paper/posesample.png?raw=true)
+
+
+----------------------------------------------------------------------------------------------------------------------------------
+## Installing the requirements
+In order to run the code you need to install python >= 3.5. 
+The requirements and the libraries needed to run the code can be installed using the following command:
+
+```
+  pip install -r requirements.txt
+```
+
+
+## Using the pre-trained models
+You can test and use the preetrained models using the following codes which are available in the following file:
+  https://github.com/aliprf/ASMNet/blob/master/main.py
+  
+```
+  tester = Test()
+  tester.test_model(ds_name=DatasetName.w300,
+                      pretrained_model_path='./pre_trained_models/ASMNet/ASM_loss/ASMNet_300W_ASMLoss.h5')
+```
+
+
+## Training Network from scratch
+
+
+### Preparing Data
+Data needs to be normalized and saved in npy format. 
+
+### PCA creation
+you can you the pca_utility.py class to create the eigenvalues, eigenvectors, and the meanvector:
+```
+pca_calc = PCAUtility()
+    pca_calc.create_pca_from_npy(dataset_name=DatasetName.w300,
+                                 labels_npy_path='./data/w300/normalized_labels/',
+                                 pca_percentages=90)
+```
+### Training 
+The training implementation is located in train.py class. You can use the following code to start the training:
+
+```
+ trainer = Train(arch=ModelArch.ASMNet,
+                    dataset_name=DatasetName.w300,
+                    save_path='./',
+                    asm_accuracy=90)
+```
+
+
+Please cite this work as:
+
+      @inproceedings{fard2021asmnet,
+            title={ASMNet: A Lightweight Deep Neural Network for Face Alignment and Pose Estimation},
+            author={Fard, Ali Pourramezan and Abdollahi, Hojjat and Mahoor, Mohammad},
+            booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
+            pages={1521--1530},
+            year={2021}
+      }
+
+```diff
+@@plaese STAR the repo if you like it.@@
+```

cnn_model.py

@@ -0,0 +1,90 @@
+from configuration import DatasetName, DatasetType, W300Conf, InputDataSize, LearningConfig
+import tensorflow as tf
+from tensorflow import keras
+from keras.regularizers import l2, l1
+
+from keras.models import Model
+from keras.applications import mobilenet_v2
+from keras.layers import Dense, MaxPooling2D, Conv2D, Flatten, \
+    BatchNormalization, GlobalAveragePooling2D, Dropout
+
+
+class CNNModel:
+    def get_model(self, arch, output_len):
+
+        if arch == 'ASMNet':
+            model = self.create_ASMNet(inp_shape=[224, 224, 3], output_len=output_len)
+
+        elif arch == 'mobileNetV2':
+            model = self.create_mobileNet(inp_shape=[224, 224, 3], output_len=output_len)
+
+        return model
+
+    def create_mobileNet(self, output_len, inp_shape):
+        mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=inp_shape,
+                                                   alpha=1.0,
+                                                   include_top=True,
+                                                   weights=None,
+                                                   pooling=None)
+        mobilenet_model.layers.pop()
+
+        x = mobilenet_model.get_layer('global_average_pooling2d_1').output  # 1280
+        out_landmarks = Dense(output_len, name='O_L')(x)
+        out_poses = Dense(LearningConfig.pose_len, name='O_P')(x)
+
+        inp = mobilenet_model.input
+        revised_model = Model(inp, [out_landmarks, out_poses])
+        revised_model.summary()
+        return revised_model
+
+    def create_ASMNet(self, output_len, inp_tensor=None, inp_shape=None):
+        mobilenet_model = mobilenet_v2.MobileNetV2(input_shape=inp_shape,
+                                                   alpha=1.0,
+                                                   include_top=True,
+                                                   weights=None,
+                                                   input_tensor=inp_tensor,
+                                                   pooling=None)
+        mobilenet_model.layers.pop()
+        inp = mobilenet_model.input
+
+        '''heatmap can not be generated from activation layers, so we use out_relu'''
+        block_1_project_BN = mobilenet_model.get_layer('block_1_project_BN').output  # 56*56*24
+        block_1_project_BN_mpool = GlobalAveragePooling2D()(block_1_project_BN)
+
+        block_3_project_BN = mobilenet_model.get_layer('block_3_project_BN').output  # 28*28*32
+        block_3_project_BN_mpool = GlobalAveragePooling2D()(block_3_project_BN)
+
+        block_6_project_BN = mobilenet_model.get_layer('block_6_project_BN').output  # 14*14*64
+        block_6_project_BN_mpool = GlobalAveragePooling2D()(block_6_project_BN)
+
+        block_10_project_BN = mobilenet_model.get_layer('block_10_project_BN').output  # 14*14*96
+        block_10_project_BN_mpool = GlobalAveragePooling2D()(block_10_project_BN)
+
+        block_13_project_BN = mobilenet_model.get_layer('block_13_project_BN').output  # 7*7*160
+        block_13_project_BN_mpool = GlobalAveragePooling2D()(block_13_project_BN)
+
+        block_15_add = mobilenet_model.get_layer('block_15_add').output  # 7*7*160
+        block_15_add_mpool = GlobalAveragePooling2D()(block_15_add)
+
+        x = keras.layers.Concatenate()([block_1_project_BN_mpool, block_3_project_BN_mpool, block_6_project_BN_mpool,
+                                        block_10_project_BN_mpool, block_13_project_BN_mpool, block_15_add_mpool])
+        x = keras.layers.Dropout(rate=0.3)(x)
+        ''''''
+        out_landmarks = Dense(output_len,
+                              kernel_regularizer=l2(0.01),
+                              bias_regularizer=l2(0.01),
+                              name='O_L')(x)
+        out_poses = Dense(LearningConfig.pose_len,
+                          kernel_regularizer=l2(0.01),
+                          bias_regularizer=l2(0.01),
+                          name='O_P')(x)
+
+        revised_model = Model(inp, [out_landmarks, out_poses])
+
+        revised_model.summary()
+        model_json = revised_model.to_json()
+
+        with open("ASMNet.json", "w") as json_file:
+            json_file.write(model_json)
+
+        return revised_model

configuration.py

@@ -0,0 +1,47 @@
+class DatasetName:
+    w300 = '300W'
+    wflw = 'wflw'
+
+
+class ModelArch:
+    ASMNet = 'ASMNet'
+    MNV2 = 'mobileNetV2'
+
+class DatasetType:
+    data_type_train = 0
+    data_type_validation = 1
+    data_type_test = 2
+
+
+class LearningConfig:
+    batch_size = 3
+    epochs = 150
+    pose_len = 3
+
+
+class InputDataSize:
+    image_input_size = 224
+    pose_len = 3
+
+
+class W300Conf:
+    W300W_prefix_path = '/media/ali/new_data/300W/'  # --> local
+
+    train_pose = W300W_prefix_path + 'train_set/pose/'
+    train_annotation = W300W_prefix_path + 'train_set/annotations/'
+    train_image = W300W_prefix_path + 'train_set/images/'
+
+    test_annotation_path = W300W_prefix_path + 'test_set/annotations/'
+    test_image_path = W300W_prefix_path + 'test_set/images/'
+    num_of_landmarks = 68
+
+class WflwConf:
+    Wflw_prefix_path = '/media/ali/new_data/wflw/'  # --> local
+
+    train_pose = Wflw_prefix_path + 'train_set/pose/'
+    train_annotation = Wflw_prefix_path + 'train_set/annotations/'
+    train_image = Wflw_prefix_path + 'train_set/images/'
+
+    test_annotation_path = Wflw_prefix_path + 'test_set/annotations/'
+    test_image_path = Wflw_prefix_path + 'test_set/images/'
+    num_of_landmarks = 98

documents/ASMNet_poster.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec90c6d8a9bfd6a424a0c6db9dc78478817b1982e81dad10a958b2195bb84e66
+size 2302669

documents/ASMNet_slides.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68a23834fb100241c8d71a424c58eb29633545055c1720fbe22b194c4bb88f24
+size 2101077

image_utility.py

@@ -0,0 +1,656 @@
+import random
+import numpy as np
+
+import matplotlib
+matplotlib.use('agg')
+import matplotlib.pyplot as plt
+
+import math
+from skimage.transform import warp, AffineTransform
+import cv2
+from scipy import misc
+from skimage.transform import rotate
+from PIL import Image
+from PIL import ImageOps
+from skimage.transform import resize
+from skimage import transform
+from skimage.transform import SimilarityTransform, AffineTransform
+import random
+from configuration import  DatasetName
+
+class ImageUtility:
+
+    def crop_and_save(self, _image, _label, file_name, num_of_landmarks, dataset_name):
+        try:
+            '''crop data: we add a small margin to the images'''
+
+            xy_points, x_points, y_points = self.create_landmarks(landmarks=_label,
+                                                                      scale_factor_x=1, scale_factor_y=1)
+
+            # self.print_image_arr(str(x_points[0]), _image, x_points, y_points)
+
+            img_arr, points_arr = self.cropImg(_image, x_points, y_points, no_padding=False)
+            # img_arr = output_img
+            # points_arr = t_label
+            '''resize image to 224*224'''
+            resized_img = resize(img_arr,
+                                 (224, 224, 3),
+                                 anti_aliasing=True)
+            dims = img_arr.shape
+            height = dims[0]
+            width = dims[1]
+            scale_factor_y = 224 / height
+            scale_factor_x = 224 / width
+
+            '''rescale and retrieve landmarks'''
+            landmark_arr_xy, landmark_arr_x, landmark_arr_y = \
+                self.create_landmarks(landmarks=points_arr,
+                                      scale_factor_x=scale_factor_x,
+                                      scale_factor_y=scale_factor_y)
+
+            min_b = 0.0
+            max_b = 224
+            if not(min(landmark_arr_x) < min_b or min(landmark_arr_y) < min_b or
+                   max(landmark_arr_x) > max_b or max(landmark_arr_y) > max_b):
+
+                # self.print_image_arr(str(landmark_arr_x[0]), resized_img, landmark_arr_x, landmark_arr_y)
+
+                im = Image.fromarray((resized_img * 255).astype(np.uint8))
+                im.save(str(file_name) + '.jpg')
+
+                pnt_file = open(str(file_name) + ".pts", "w")
+                pre_txt = ["version: 1 \n", "n_points: 68 \n", "{ \n"]
+                pnt_file.writelines(pre_txt)
+                points_txt = ""
+                for i in range(0, len(landmark_arr_xy), 2):
+                    points_txt += str(landmark_arr_xy[i]) + " " + str(landmark_arr_xy[i + 1]) + "\n"
+
+                pnt_file.writelines(points_txt)
+                pnt_file.write("} \n")
+                pnt_file.close()
+
+        except Exception as e:
+            print(e)
+
+    def random_rotate(self, _image, _label, file_name, num_of_landmarks, dataset_name):
+        try:
+
+            xy_points, x_points, y_points = self.create_landmarks(landmarks=_label,
+                                                                  scale_factor_x=1, scale_factor_y=1)
+            # self.print_image_arr(str(xy_points[8]), _image, x_points, y_points)
+
+            _image, _label = self.cropImg_2time(_image, x_points, y_points)
+
+            _image = self.__noisy(_image)
+
+            scale = (np.random.uniform(0.8, 1.0), np.random.uniform(0.8, 1.0))
+            # scale = (1, 1)
+
+            rot = np.random.uniform(-1 * 0.55, 0.55)
+            translation = (0, 0)
+            shear = 0
+
+            tform = AffineTransform(
+                scale=scale,  # ,
+                rotation=rot,
+                translation=translation,
+                shear=np.deg2rad(shear)
+            )
+
+            output_img = transform.warp(_image, tform.inverse, mode='symmetric')
+
+            sx, sy = scale
+            t_matrix = np.array([
+                [sx * math.cos(rot), -sy * math.sin(rot + shear), 0],
+                [sx * math.sin(rot), sy * math.cos(rot + shear), 0],
+                [0, 0, 1]
+            ])
+            landmark_arr_xy, landmark_arr_x, landmark_arr_y = self.create_landmarks(_label, 1, 1)
+            label = np.array(landmark_arr_x + landmark_arr_y).reshape([2, num_of_landmarks])
+            marging = np.ones([1, num_of_landmarks])
+            label = np.concatenate((label, marging), axis=0)
+
+            label_t = np.dot(t_matrix, label)
+            lbl_flat = np.delete(label_t, 2, axis=0).reshape([2*num_of_landmarks])
+
+            t_label = self.__reorder(lbl_flat, num_of_landmarks)
+
+            '''crop data: we add a small margin to the images'''
+            xy_points, x_points, y_points = self.create_landmarks(landmarks=t_label,
+                                                                  scale_factor_x=1, scale_factor_y=1)
+            img_arr, points_arr = self.cropImg(output_img, x_points, y_points, no_padding=False)
+            # img_arr = output_img
+            # points_arr = t_label
+            '''resize image to 224*224'''
+            resized_img = resize(img_arr,
+                                 (224, 224, 3),
+                                 anti_aliasing=True)
+            dims = img_arr.shape
+            height = dims[0]
+            width = dims[1]
+            scale_factor_y = 224 / height
+            scale_factor_x = 224 / width
+
+            '''rescale and retrieve landmarks'''
+            landmark_arr_xy, landmark_arr_x, landmark_arr_y = \
+                self.create_landmarks(landmarks=points_arr,
+                                      scale_factor_x=scale_factor_x,
+                                      scale_factor_y=scale_factor_y)
+
+            min_b = 0.0
+            max_b = 224
+            if dataset_name == DatasetName.cofw:
+                min_b = 5.0
+                max_b = 214
+
+            if not(min(landmark_arr_x) < 0 or min(landmark_arr_y) < min_b or
+                   max(landmark_arr_x) > 224 or max(landmark_arr_y) > max_b):
+
+                # self.print_image_arr(str(landmark_arr_x[0]), resized_img, landmark_arr_x, landmark_arr_y)
+
+                im = Image.fromarray((resized_img * 255).astype(np.uint8))
+                im.save(str(file_name) + '.jpg')
+
+                pnt_file = open(str(file_name) + ".pts", "w")
+                pre_txt = ["version: 1 \n", "n_points: 68 \n", "{ \n"]
+                pnt_file.writelines(pre_txt)
+                points_txt = ""
+                for i in range(0, len(landmark_arr_xy), 2):
+                    points_txt += str(landmark_arr_xy[i]) + " " + str(landmark_arr_xy[i + 1]) + "\n"
+
+                pnt_file.writelines(points_txt)
+                pnt_file.write("} \n")
+                pnt_file.close()
+
+            return t_label, output_img
+        except Exception as e:
+            print(e)
+            return None, None
+
+
+    def random_rotate_m(self, _image, _label_img, file_name):
+
+        rot = random.uniform(-80.9, 80.9)
+
+        output_img = rotate(_image, rot, resize=True)
+        output_img_lbl = rotate(_label_img, rot, resize=True)
+
+        im = Image.fromarray((output_img * 255).astype(np.uint8))
+        im_lbl = Image.fromarray((output_img_lbl * 255).astype(np.uint8))
+
+        im_m = ImageOps.mirror(im)
+        im_lbl_m = ImageOps.mirror(im_lbl)
+
+        im.save(str(file_name)+'.jpg')
+        # im_lbl.save(str(file_name)+'_lbl.jpg')
+
+        im_m.save(str(file_name) + '_m.jpg')
+        # im_lbl_m.save(str(file_name) + '_m_lbl.jpg')
+
+        im_lbl_ar = np.array(im_lbl)
+        im_lbl_m_ar = np.array(im_lbl_m)
+
+        self.__save_label(im_lbl_ar, file_name, np.array(im))
+        self.__save_label(im_lbl_m_ar, file_name+"_m", np.array(im_m))
+
+
+    def __save_label(self, im_lbl_ar, file_name, img_arr):
+
+        im_lbl_point = []
+        for i in range(im_lbl_ar.shape[0]):
+            for j in range(im_lbl_ar.shape[1]):
+                if im_lbl_ar[i, j] != 0:
+                    im_lbl_point.append(j)
+                    im_lbl_point.append(i)
+
+        pnt_file = open(str(file_name)+".pts", "w")
+
+        pre_txt = ["version: 1 \n", "n_points: 68 \n", "{ \n"]
+        pnt_file.writelines(pre_txt)
+        points_txt = ""
+        for i in range(0, len(im_lbl_point), 2):
+            points_txt += str(im_lbl_point[i]) + " " + str(im_lbl_point[i+1]) + "\n"
+
+        pnt_file.writelines(points_txt)
+        pnt_file.write("} \n")
+        pnt_file.close()
+
+        '''crop data: we add a small margin to the images'''
+        xy_points, x_points, y_points = self.create_landmarks(landmarks=im_lbl_point,
+                                                                       scale_factor_x=1, scale_factor_y=1)
+        img_arr, points_arr = self.cropImg(img_arr, x_points, y_points)
+
+        '''resize image to 224*224'''
+        resized_img = resize(img_arr,
+                             (224, 224, 3),
+                             anti_aliasing=True)
+        dims = img_arr.shape
+        height = dims[0]
+        width = dims[1]
+        scale_factor_y = 224 / height
+        scale_factor_x = 224 / width
+
+        '''rescale and retrieve landmarks'''
+        landmark_arr_xy, landmark_arr_x, landmark_arr_y = \
+            self.create_landmarks(landmarks=points_arr,
+                                           scale_factor_x=scale_factor_x,
+                                           scale_factor_y=scale_factor_y)
+
+        im = Image.fromarray((resized_img * 255).astype(np.uint8))
+        im.save(str(im_lbl_point[0])+'.jpg')
+        # self.print_image_arr(im_lbl_point[0], resized_img, landmark_arr_x, landmark_arr_y)
+
+
+    def augment(self, _image, _label, num_of_landmarks):
+
+        # face = misc.face(gray=True)
+        #
+        # rotate_face = ndimage.rotate(_image, 45)
+        # self.print_image_arr(_label[0], rotate_face, [],[])
+
+        # hue_img = tf.image.random_hue(_image, max_delta=0.1)  # max_delta must be in the interval [0, 0.5].
+        # sat_img = tf.image.random_saturation(hue_img, lower=0.0, upper=3.0)
+        #
+        # sat_img = K.eval(sat_img)
+        #
+        _image = self.__noisy(_image)
+
+        shear = 0
+
+        # rot = 0.0
+        '''this scale has problem'''
+        # scale = (random.uniform(0.8, 1.00), random.uniform(0.8, 1.00))
+
+        scale = (1, 1)
+
+        rot = np.random.uniform(-1 * 0.008, 0.008)
+
+        tform = AffineTransform(scale=scale, rotation=rot, shear=shear,
+                                translation=(0, 0))
+
+        output_img = warp(_image, tform.inverse, output_shape=(_image.shape[0], _image.shape[1]))
+
+        sx, sy = scale
+        t_matrix = np.array([
+            [sx * math.cos(rot), -sy * math.sin(rot + shear), 0],
+            [sx * math.sin(rot), sy * math.cos(rot + shear), 0],
+            [0, 0, 1]
+        ])
+        landmark_arr_xy, landmark_arr_x, landmark_arr_y = self.create_landmarks(_label, 1, 1)
+        label = np.array(landmark_arr_x + landmark_arr_y).reshape([2, num_of_landmarks])
+        marging = np.ones([1, num_of_landmarks])
+        label = np.concatenate((label, marging), axis=0)
+
+        label_t = np.dot(t_matrix, label)
+        lbl_flat = np.delete(label_t, 2, axis=0).reshape([num_of_landmarks*2])
+
+        t_label = self.__reorder(lbl_flat, num_of_landmarks)
+        return t_label, output_img
+
+    def __noisy(self, image):
+        noise_typ = random.randint(0, 5)
+        # if True or noise_typ == 0 :#"gauss":
+        #     row, col, ch = image.shape
+        #     mean = 0
+        #     var = 0.001
+        #     sigma = var ** 0.1
+        #     gauss = np.random.normal(mean, sigma, (row, col, ch))
+        #     gauss = gauss.reshape(row, col, ch)
+        #     noisy = image + gauss
+        #     return noisy
+        if 1 <= noise_typ <= 2:# "s&p":
+            row, col, ch = image.shape
+            s_vs_p = 0.5
+            amount = 0.04
+            out = np.copy(image)
+            # Salt mode
+            num_salt = np.ceil(amount * image.size * s_vs_p)
+            coords = [np.random.randint(0, i - 1, int(num_salt))
+                      for i in image.shape]
+            out[coords] = 1
+
+            # Pepper mode
+            num_pepper = np.ceil(amount * image.size * (1. - s_vs_p))
+            coords = [np.random.randint(0, i - 1, int(num_pepper))
+                      for i in image.shape]
+            out[coords] = 0
+            return out
+
+        # elif 5 <=noise_typ <= 7: #"speckle":
+        #     row, col, ch = image.shape
+        #     gauss = np.random.randn(row, col, ch)
+        #     gauss = gauss.reshape(row, col, ch)
+        #     noisy = image + image * gauss
+        #     return noisy
+        else:
+            return image
+
+    def __reorder(self, input_arr, num_of_landmarks):
+        out_arr = []
+        for i in range(num_of_landmarks):
+            out_arr.append(input_arr[i])
+            k = num_of_landmarks + i
+            out_arr.append(input_arr[k])
+        return np.array(out_arr)
+
+    def print_image_arr_heat(self, k, image):
+        plt.figure()
+        plt.imshow(image)
+        implot = plt.imshow(image)
+        plt.axis('off')
+        plt.savefig('heat' + str(k) + '.png', bbox_inches='tight')
+        plt.clf()
+
+    def print_image_arr(self, k, image, landmarks_x, landmarks_y):
+        plt.figure()
+        plt.imshow(image)
+        implot = plt.imshow(image)
+
+        plt.scatter(x=landmarks_x[:], y=landmarks_y[:], c='black', s=20)
+        plt.scatter(x=landmarks_x[:], y=landmarks_y[:], c='white', s=15)
+        plt.axis('off')
+        plt.savefig('sss' + str(k) + '.png',  bbox_inches='tight')
+        # plt.show()
+        plt.clf()
+
+    def create_landmarks_from_normalized_original_img(self, img, landmarks, width, height, x_center, y_center, x1, y1, scale_x, scale_y):
+        # landmarks_splited = _landmarks.split(';')
+        landmark_arr_xy = []
+        landmark_arr_x = []
+        landmark_arr_y = []
+
+        for j in range(0, len(landmarks), 2):
+            x = ((x_center - float(landmarks[j]) * width)*scale_x) + x1
+            y = ((y_center - float(landmarks[j + 1]) * height)*scale_y) + y1
+
+            landmark_arr_xy.append(x)
+            landmark_arr_xy.append(y)
+
+            landmark_arr_x.append(x)
+            landmark_arr_y.append(y)
+
+            img = cv2.circle(img, (int(x), int(y)), 2, (255, 14, 74), 2)
+            img = cv2.circle(img, (int(x), int(y)), 1, (0, 255, 255), 1)
+
+        return landmark_arr_xy, landmark_arr_x, landmark_arr_y, img
+
+
+    def create_landmarks_from_normalized(self, landmarks, width, height, x_center, y_center):
+
+        # landmarks_splited = _landmarks.split(';')
+        landmark_arr_xy = []
+        landmark_arr_x = []
+        landmark_arr_y = []
+
+        for j in range(0, len(landmarks), 2):
+            x = x_center - float(landmarks[j]) * width
+            y = y_center - float(landmarks[j + 1]) * height
+
+            landmark_arr_xy.append(x)
+            landmark_arr_xy.append(y)  # [ x1, y1, x2,y2 ]
+
+            landmark_arr_x.append(x)  # [x1, x2]
+            landmark_arr_y.append(y)  # [y1, y2]
+
+        return landmark_arr_xy, landmark_arr_x, landmark_arr_y
+
+    def create_landmarks(self, landmarks, scale_factor_x, scale_factor_y):
+        # landmarks_splited = _landmarks.split(';')
+        landmark_arr_xy = []
+        landmark_arr_x = []
+        landmark_arr_y = []
+        for j in range(0, len(landmarks), 2):
+
+            x = float(landmarks[j]) * scale_factor_x
+            y = float(landmarks[j + 1]) * scale_factor_y
+
+            landmark_arr_xy.append(x)
+            landmark_arr_xy.append(y)  # [ x1, y1, x2,y2 ]
+
+            landmark_arr_x.append(x)  # [x1, x2]
+            landmark_arr_y.append(y)  # [y1, y2]
+
+        return landmark_arr_xy, landmark_arr_x, landmark_arr_y
+
+    def create_landmarks_aflw(self, landmarks, scale_factor_x, scale_factor_y):
+        # landmarks_splited = _landmarks.split(';')
+        landmark_arr_xy = []
+        landmark_arr_x = []
+        landmark_arr_y = []
+        for j in range(0, len(landmarks), 2):
+            if landmarks[j][0] == 1:
+                x = float(landmarks[j][1]) * scale_factor_x
+                y = float(landmarks[j][2]) * scale_factor_y
+
+                landmark_arr_xy.append(x)
+                landmark_arr_xy.append(y)  # [ x1, y1, x2,y2 ]
+
+                landmark_arr_x.append(x)  # [x1, x2]
+                landmark_arr_y.append(y)  # [y1, y2]
+
+        return landmark_arr_xy, landmark_arr_x, landmark_arr_y
+
+    def random_augmentation(self, lbl, img, number_of_landmark):
+        # a = random.randint(0, 2)
+        # if a == 0:
+        #     img, lbl = self.__add_margin(img, img.shape[0], lbl)
+
+        '''this function has problem!!!'''
+        # img, lbl = self.__add_margin(img, img.shape[0], lbl)
+
+        # else:
+        #     img, lbl = self.__negative_crop(img, lbl)
+
+        # i = random.randint(0, 2)
+        # if i == 0:
+        #     img, lbl = self.__rotate(img, lbl, 90, img.shape[0], img.shape[1])
+        # elif i == 1:
+        #     img, lbl = self.__rotate(img, lbl, 180, img.shape[0], img.shape[1])
+        # else:
+        #     img, lbl = self.__rotate(img, lbl, 270, img.shape[0], img.shape[1])
+
+        # k = random.randint(0, 3)
+        # if k > 0:
+        #     img = self.__change_color(img)
+        #
+        img = self.__noisy(img)
+
+        lbl = np.reshape(lbl, [number_of_landmark*2])
+        return lbl, img
+
+
+    def cropImg_2time(self, img, x_s, y_s):
+        min_x = max(0, int(min(x_s) - 100))
+        max_x = int(max(x_s) + 100)
+        min_y = max(0, int(min(y_s) - 100))
+        max_y = int(max(y_s) + 100)
+
+        crop = img[min_y:max_y, min_x:max_x]
+
+        new_x_s = []
+        new_y_s = []
+        new_xy_s = []
+
+        for i in range(len(x_s)):
+            new_x_s.append(x_s[i] - min_x)
+            new_y_s.append(y_s[i] - min_y)
+            new_xy_s.append(x_s[i] - min_x)
+            new_xy_s.append(y_s[i] - min_y)
+        return crop, new_xy_s
+
+    def cropImg(self, img, x_s, y_s, no_padding=False):
+        margin1 = random.randint(0, 10)
+        margin2 = random.randint(0, 10)
+        margin3 = random.randint(0, 10)
+        margin4 = random.randint(0, 10)
+
+        if no_padding:
+            min_x = max(0, int(min(x_s)))
+            max_x = int(max(x_s))
+            min_y = max(0, int(min(y_s)))
+            max_y = int(max(y_s))
+        else:
+            min_x = max(0, int(min(x_s) - margin1))
+            max_x = int(max(x_s) + margin2)
+            min_y = max(0, int(min(y_s) - margin3))
+            max_y = int(max(y_s) + margin4)
+
+        crop = img[min_y:max_y, min_x:max_x]
+
+        new_x_s = []
+        new_y_s = []
+        new_xy_s = []
+
+        for i in range(len(x_s)):
+            new_x_s.append(x_s[i] - min_x)
+            new_y_s.append(y_s[i] - min_y)
+            new_xy_s.append(x_s[i] - min_x)
+            new_xy_s.append(y_s[i] - min_y)
+
+        # imgpr.print_image_arr(k, crop, new_x_s, new_y_s)
+        # imgpr.print_image_arr_2(i, img, x_s, y_s, [min_x, max_x], [min_y, max_y])
+
+        return crop, new_xy_s
+
+    def __negative_crop(self, img, landmarks):
+
+        landmark_arr_xy, x_s, y_s = self.create_landmarks(landmarks, 1, 1)
+        min_x = img.shape[0] // random.randint(5, 15)
+        max_x = img.shape[0] - (img.shape[0] // random.randint(15, 20))
+        min_y = img.shape[0] // random.randint(5, 15)
+        max_y = img.shape[0] - (img.shape[0] // random.randint(15, 20))
+
+        crop = img[min_y:max_y, min_x:max_x]
+
+        new_x_s = []
+        new_y_s = []
+        new_xy_s = []
+
+        for i in range(len(x_s)):
+            new_x_s.append(x_s[i] - min_x)
+            new_y_s.append(y_s[i] - min_y)
+            new_xy_s.append(x_s[i] - min_x)
+            new_xy_s.append(y_s[i] - min_y)
+
+        # imgpr.print_image_arr(crop.shape[0], crop, new_x_s, new_y_s)
+        # imgpr.print_image_arr_2(crop.shape[0], crop, x_s, y_s, [min_x, max_x], [min_y, max_y])
+
+        return crop, new_xy_s
+
+    def __add_margin(self, img, img_w, lbl):
+        marging_width = img_w // random.randint(15, 20)
+        direction = random.randint(0, 4)
+
+        if direction == 1:
+            margings = np.random.random([img_w, int(marging_width), 3])
+            img = np.concatenate((img, margings), axis=1)
+
+        if direction == 2:
+            margings_1 = np.random.random([img_w, int(marging_width), 3])
+            img = np.concatenate((img, margings_1), axis=1)
+
+            marging_width_1 = img_w // random.randint(15, 20)
+            margings_2 = np.random.random([int(marging_width_1), img_w + int(marging_width), 3])
+            img = np.concatenate((img, margings_2), axis=0)
+
+        if direction == 3:  # need chane labels
+            margings_1 = np.random.random([img_w, int(marging_width), 3])
+            img = np.concatenate((margings_1, img), axis=1)
+            lbl = self.__transfer_lbl(int(marging_width), lbl, [1, 0])
+
+            marging_width_1 = img_w // random.randint(15, 20)
+            margings_2 = np.random.random([int(marging_width_1), img_w + int(marging_width), 3])
+            img = np.concatenate((margings_2, img), axis=0)
+            lbl = self.__transfer_lbl(int(marging_width_1), lbl, [0, 1])
+
+        if direction == 4:  # need chane labels
+            margings_1 = np.random.random([img_w, int(marging_width), 3])
+            img = np.concatenate((margings_1, img), axis=1)
+            lbl = self.__transfer_lbl(int(marging_width), lbl, [1, 0])
+            img_w1 = img_w + int(marging_width)
+
+            marging_width_1 = img_w // random.randint(15, 20)
+            margings_2 = np.random.random([int(marging_width_1), img_w1, 3])
+            img = np.concatenate((margings_2, img), axis=0)
+            lbl = self.__transfer_lbl(int(marging_width_1), lbl, [0, 1])
+            img_w2 = img_w + int(marging_width_1)
+
+            marging_width_1 = img_w // random.randint(15, 20)
+            margings_1 = np.random.random([img_w2, int(marging_width_1), 3])
+            img = np.concatenate((img, margings_1), axis=1)
+
+            marging_width_1 = img_w // random.randint(15, 20)
+            margings_2 = np.random.random([int(marging_width_1), img.shape[1], 3])
+            img = np.concatenate((img, margings_2), axis=0)
+
+        return img, lbl
+
+    def __void_image(self, img, img_w, ):
+        marging_width = int(img_w / random.randint(7, 16))
+        direction = random.randint(0, 1)
+        direction = 0
+        if direction == 0:
+            np.delete(img, 100, 1)
+            # img[:, 0:marging_width, :] = 0
+        elif direction == 1:
+            img[img_w - marging_width:img_w, :, :] = 0
+        if direction == 2:
+            img[:, img_w - marging_width:img_w, :] = 0
+
+        return img
+
+    def __change_color(self, img):
+        # color_arr = np.random.random([img.shape[0], img.shape[1]])
+        color_arr = np.zeros([img.shape[0], img.shape[1]])
+        axis = random.randint(0, 4)
+
+        if axis == 0:  # red
+            img_mono = img[:, :, 0]
+            new_img = np.stack([img_mono, color_arr, color_arr], axis=2)
+        elif axis == 1:  # green
+            img_mono = img[:, :, 1]
+            new_img = np.stack([color_arr, img_mono, color_arr], axis=2)
+        elif axis == 2:  # blue
+            img_mono = img[:, :, 1]
+            new_img = np.stack([color_arr, img_mono, color_arr], axis=2)
+        elif axis == 3:  # gray scale
+            img_mono = img[:, :, 0]
+            new_img = np.stack([img_mono, img_mono, img_mono], axis=2)
+        else:  # random noise
+            color_arr = np.random.random([img.shape[0], img.shape[1]])
+            img_mono = img[:, :, 0]
+            new_img = np.stack([img_mono, img_mono, color_arr], axis=2)
+
+        return new_img
+
+    def __rotate_origin_only(self, xy_arr, radians, xs, ys):
+        """Only rotate a point around the origin (0, 0)."""
+        rotated = []
+        for xy in xy_arr:
+            x, y = xy
+            xx = x * math.cos(radians) + y * math.sin(radians)
+            yy = -x * math.sin(radians) + y * math.cos(radians)
+            rotated.append([xx + xs, yy + ys])
+        return np.array(rotated)
+
+    def __rotate(self, img, landmark_old, degree, img_w, img_h, num_of_landmarks):
+        landmark_old = np.reshape(landmark_old, [num_of_landmarks, 2])
+
+        theta = math.radians(degree)
+
+        if degree == 90:
+            landmark = self.__rotate_origin_only(landmark_old, theta, 0, img_h)
+            return np.rot90(img, 3, axes=(-2, 0)), landmark
+        elif degree == 180:
+            landmark = self.__rotate_origin_only(landmark_old, theta, img_h, img_w)
+            return np.rot90(img, 2, axes=(-2, 0)), landmark
+        elif degree == 270:
+            landmark = self.__rotate_origin_only(landmark_old, theta, img_w, 0)
+            return np.rot90(img, 1, axes=(-2, 0)), landmark
+
+    def __transfer_lbl(self, marging_width_1, lbl, axis_arr):
+        new_lbl = []
+        for i in range(0, len(lbl), 2):
+            new_lbl.append(lbl[i] + marging_width_1 * axis_arr[0])
+            new_lbl.append(lbl[i + 1] + marging_width_1 * axis_arr[1])
+        return np.array(new_lbl)

main.py

@@ -0,0 +1,23 @@
+from train import Train
+from test import Test
+from configuration import DatasetName, ModelArch
+from pca_utility import PCAUtility
+if __name__ == '__main__':
+    '''use the pretrained model'''
+    tester = Test()
+    tester.test_model(ds_name=DatasetName.w300,
+                      pretrained_model_path='./pre_trained_models/ASMNet/ASM_loss/ASMNet_300W_ASMLoss.h5')
+
+    '''training model from scratch'''
+    #   pretrain prerequisites
+    #       1- PCA calculation:
+    pca_calc = PCAUtility()
+    pca_calc.create_pca_from_npy(dataset_name=DatasetName.w300,
+                                 labels_npy_path='./data/w300/normalized_labels/',
+                                 pca_percentages=90)
+
+    #  Train:
+    trainer = Train(arch=ModelArch.ASMNet,
+                    dataset_name=DatasetName.w300,
+                    save_path='./',
+                    asm_accuracy=90)

pca_utility.py

@@ -0,0 +1,72 @@
+from configuration import DatasetName, DatasetType, W300Conf, InputDataSize, LearningConfig, WflwConf
+from image_utility import ImageUtility
+from sklearn.decomposition import PCA, IncrementalPCA
+from sklearn.decomposition import TruncatedSVD
+import numpy as np
+import pickle
+import os
+from tqdm import tqdm
+from numpy import save, load
+import math
+from PIL import Image
+from numpy import save, load
+
+
+class PCAUtility:
+    eigenvalues_prefix = "_eigenvalues_"
+    eigenvectors_prefix = "_eigenvectors_"
+    meanvector_prefix = "_meanvector_"
+
+
+
+    def create_pca_from_npy(self, dataset_name, labels_npy_path, pca_percentages):
+        """
+        generate and save eigenvalues, eigenvectors, meanvector
+        :param labels_npy_path: the path to the normalized labels that are save in npy format.
+        :param pca_percentages: % of eigenvalues that will be used
+        :return: generate
+        """
+        path = labels_npy_path
+        print('PCA calculation started: loading labels')
+
+        lbl_arr = []
+        for file in tqdm(os.listdir(path)):
+            if file.endswith(".npy"):
+                npy_file = os.path.join(path, file)
+                lbl_arr.append(load(npy_file))
+
+        lbl_arr = np.array(lbl_arr)
+
+        reduced_lbl_arr, eigenvalues, eigenvectors = self._func_PCA(lbl_arr, pca_percentages)
+        mean_lbl_arr = np.mean(lbl_arr, axis=0)
+        eigenvectors = eigenvectors.T
+
+        save('./pca_obj/' + dataset_name + self.eigenvalues_prefix + str(pca_percentages), eigenvalues)
+        save('./pca_obj/' + dataset_name + self.eigenvectors_prefix + str(pca_percentages), eigenvectors)
+        save('./pca_obj/' + dataset_name + self.meanvector_prefix + str(pca_percentages), mean_lbl_arr)
+
+    def load_pca_obj(self, dataset_name, pca_percentages):
+        eigenvalues = np.load('./pca_obj/' + dataset_name + self.eigenvalues_prefix + str(pca_percentages))
+        eigenvectors = np.load('./pca_obj/' + dataset_name + self.eigenvectors_prefix + str(pca_percentages))
+        meanvector = np.load('./pca_obj/' + dataset_name + self.meanvector_prefix + str(pca_percentages))
+        return eigenvalues, eigenvectors, meanvector
+
+    def _func_PCA(self, input_data, pca_postfix):
+        input_data = np.array(input_data)
+        pca = PCA(n_components=pca_postfix / 100)
+        # pca = PCA(n_components=0.98)
+        # pca = IncrementalPCA(n_components=50, batch_size=50)
+        pca.fit(input_data)
+        pca_input_data = pca.transform(input_data)
+        eigenvalues = pca.explained_variance_
+        eigenvectors = pca.components_
+        return pca_input_data, eigenvalues, eigenvectors
+
+    def __svd_func(self, input_data, pca_postfix):
+        svd = TruncatedSVD(n_components=50)
+        svd.fit(input_data)
+        pca_input_data = svd.transform(input_data)
+        eigenvalues = svd.explained_variance_
+        eigenvectors = svd.components_
+        return pca_input_data, eigenvalues, eigenvectors
+        # U, S, VT = svd(input_data)

pre_trained_models/ASMNet/ASM_loss/ASMNet_300W_ASMLoss.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f12bcb0cc89d1f83b80ead7757f351113f0a919b31c14e5c153f88d7a4fc1d1
+size 17690416

pre_trained_models/ASMNet/ASM_loss/ASMNet_WFLW_ASMLoss.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0eb70af0c1be3b7214889523e599c9ceedb0041029fb3c08ab41a73697777f78
+size 18076832

pre_trained_models/ASMNet/MSE_loss/ASMNet_300W_MESLoss.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9c072f7e0c2b3482ff42592db49693c5eaf272beaec0c9e0ed5c212c10e7fb3
+size 17690416

pre_trained_models/ASMNet/MSE_loss/ASMNet_WFLW_MESLoss.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58dab640965d280c19251fc8331d6ad8a3a693ec96625ca47b1658ece156242a
+size 18076832

pre_trained_models/MobileNetV2/ASM_loss/MobileNetV2_300W_ASMLoss.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3293865821660e229edfb11e9c5e2d0eab75c54aedbf8e14fb3801076b5712bc
+size 29631480

pre_trained_models/MobileNetV2/ASM_loss/MobileNetV2_WFLW_ASMLoss.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62fb8a844437d56c9b9f2d05ba0bc23eaeff40871a5bbfa898348eec4cafa4cd
+size 30551608

pre_trained_models/MobileNetV2/MSE_loss/MobileNetV2_300W_MESLoss.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:452739aab8ec9d3e6a2121325015cf78cb0e3720c968710019896052769d005c
+size 29631480

pre_trained_models/MobileNetV2/MSE_loss/MobileNetV2_WFLW_MESLoss.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62abc61af5ae022895ea7619419ff8b11d3f83f5130368243b29d3d55c1a9700
+size 30551608

requirements.txt

@@ -0,0 +1,23 @@
+# for cuda 9
+# pip install torch==1.2.0+cu92 torchvision==0.4.0+cu92 -f https://download.pytorch.org/whl/torch_stable.html
+#
+#torch
+#torchvision
+bottleneck
+numpy==1.19.2
+#tensorflow==1.14.0
+tensorflow==2.3.1
+#tensorflow-gpu==1.14
+# keras==2.2.4
+keras==2.4.3
+matplotlib
+opencv-python
+opencv-contrib-python
+scipy
+scikit-learn
+scikit-image
+Pillow
+tqdm
+efficientnet
+# tfkerassurgeon
+tensorboard

test.py

@@ -0,0 +1,40 @@
+from configuration import DatasetName, WflwConf, W300Conf, DatasetType, LearningConfig, InputDataSize
+import tensorflow as tf
+
+import cv2
+import os.path
+import scipy.io as sio
+from cnn_model import CNNModel
+from tqdm import tqdm
+import numpy as np
+from os import listdir
+from os.path import isfile, join
+from scipy.integrate import simps
+from scipy.integrate import trapz
+import matplotlib.pyplot as plt
+from skimage.io import imread
+
+class Test:
+    def test_model(self, pretrained_model_path, ds_name):
+        if ds_name == DatasetName.w300:
+            test_annotation_path = W300Conf.test_annotation_path
+            test_image_path = W300Conf.test_image_path
+        elif ds_name == DatasetName.wflw:
+            test_annotation_path = WflwConf.test_annotation_path
+            test_image_path = WflwConf.test_image_path
+
+        model = tf.keras.models.load_model(pretrained_model_path)
+
+        for i, file in tqdm(enumerate(os.listdir(test_image_path))):
+            # load image and then normalize it
+            img = imread(test_image_path + file)/255.0
+
+            # prediction
+            prediction = model.predict(np.expand_dims(img, axis=0))
+
+            # the first dimension is landmark point
+            landmark_predicted = prediction[0][0]
+
+            # the second dimension is the pose
+            pose_predicted = prediction[1][0]
+

train.py

@@ -0,0 +1,207 @@
+from configuration import DatasetName, WflwConf, W300Conf, DatasetType, LearningConfig, InputDataSize
+from cnn_model import CNNModel
+import tensorflow as tf
+import tensorflow.keras as keras
+import numpy as np
+import matplotlib.pyplot as plt
+import math
+from datetime import datetime
+from sklearn.utils import shuffle
+from sklearn.model_selection import train_test_split
+from numpy import save, load, asarray
+import csv
+from skimage.io import imread
+import pickle
+from image_utility import ImageUtility
+from tqdm import tqdm
+import os
+from Asm_assisted_loss import ASMLoss
+from cnn_model import CNNModel
+
+
+class Train:
+    def __init__(self, arch, dataset_name, save_path, asm_accuracy=90):
+        """
+        :param arch:
+        :param dataset_name:
+        :param save_path:
+        :param asm_accuracy:
+        """
+
+        self.dataset_name = dataset_name
+        self.save_path = save_path
+        self.arch = arch
+        self.asm_accuracy = asm_accuracy
+
+        if dataset_name == DatasetName.w300:
+            self.num_landmark = W300Conf.num_of_landmarks * 2
+            self.img_path = W300Conf.train_image
+            self.annotation_path = W300Conf.train_annotation
+            self.pose_path = W300Conf.train_pose
+
+        if dataset_name == DatasetName.wflw:
+            self.num_landmark = WflwConf.num_of_landmarks * 2
+            self.img_path = WflwConf.train_image
+            self.annotation_path = WflwConf.train_annotation
+            self.pose_path = WflwConf.train_pose
+
+    def train(self, weight_path):
+        """
+
+        :param weight_path:
+        :return:
+        """
+
+        '''create loss'''
+        c_loss = ASMLoss(dataset_name=self.dataset_name, accuracy=90)
+        cnn = CNNModel()
+        '''making models'''
+        model = cnn.get_model(arch=self.arch, output_len=self.num_landmark)
+        if weight_path is not None:
+            model.load_weights(weight_path)
+
+        '''create sample generator'''
+        image_names, landmark_names, pose_names = self._create_generators()
+
+        '''create train configuration'''
+        step_per_epoch = len(image_names) // LearningConfig.batch_size
+
+        '''start train:'''
+        optimizer = tf.keras.optimizers.Adam(lr=1e-2, decay=1e-5)
+        for epoch in range(LearningConfig.epochs):
+            image_names, landmark_names, pose_names = shuffle(image_names, landmark_names, pose_names)
+            for batch_index in range(step_per_epoch):
+                '''load annotation and images'''
+                images, annotation_gr, poses_gr = self._get_batch_sample(
+                    batch_index=batch_index,
+                    img_filenames=image_names,
+                    landmark_filenames=landmark_names,
+                    pose_filenames=pose_names)
+
+                '''convert to tensor'''
+                images = tf.cast(images, tf.float32)
+                annotation_gr = tf.cast(annotation_gr, tf.float32)
+                poses_gr = tf.cast(poses_gr, tf.float32)
+
+                '''train step'''
+                self.train_step(epoch=epoch,
+                                step=batch_index,
+                                total_steps=step_per_epoch,
+                                model=model,
+                                images=images,
+                                annotation_gt=annotation_gr,
+                                poses_gt=poses_gr,
+                                optimizer=optimizer,
+                                c_loss=c_loss)
+            '''save weights'''
+            model.save(self.save_path + self.arch + str(epoch) + '_' + self.dataset_name)
+
+    def train_step(self, epoch, step, total_steps, model, images, annotation_gt, poses_gt, optimizer, c_loss):
+        """
+
+        :param epoch:
+        :param step:
+        :param total_steps:
+        :param model:
+        :param images:
+        :param annotation_gt:
+        :param poses_gt:
+        :param optimizer:
+        :param c_loss:
+        :return:
+        """
+
+        with tf.GradientTape() as tape:
+            '''create annotation_predicted'''
+            annotation_predicted, pose_predicted = model(images, training=True)
+            '''calculate loss'''
+            mse_loss, asm_loss = c_loss.calculate_landmark_ASM_assisted_loss(landmark_pr=annotation_predicted,
+                                                                             landmark_gt=annotation_gt,
+                                                                             current_epoch=epoch,
+                                                                             total_steps=total_steps)
+            pose_loss = c_loss.calculate_pose_loss(x_pr=pose_predicted, x_gt=poses_gt)
+
+            '''calculate loss'''
+            total_loss = mse_loss + asm_loss + pose_loss
+
+        '''calculate gradient'''
+        gradients_of_model = tape.gradient(total_loss, model.trainable_variables)
+        '''apply Gradients:'''
+        optimizer.apply_gradients(zip(gradients_of_model, model.trainable_variables))
+        '''printing loss Values: '''
+        tf.print("->EPOCH: ", str(epoch), "->STEP: ", str(step) + '/' + str(total_steps), ' -> : total_loss: ',
+                 total_loss)
+
+    def _create_generators(self):
+        """
+        :return:
+        """
+        image_names, landmark_filenames, pose_names = \
+            self._create_image_and_labels_name(img_path=self.img_path,
+                                               annotation_path=self.annotation_path,
+                                               pose_path=self.pose_path)
+        return image_names, landmark_filenames, pose_names
+
+    def _create_image_and_labels_name(self, img_path, annotation_path, pose_path):
+        """
+
+        :param img_path:
+        :param annotation_path:
+        :param pose_path:
+        :return:
+        """
+        img_filenames = []
+        landmark_filenames = []
+        poses_filenames = []
+
+        for file in os.listdir(img_path):
+            if file.endswith(".jpg") or file.endswith(".png"):
+                lbl_file = str(file)[:-3] + "npy"  # just name
+                pose_file = str(file)[:-3] + "npy"  # just name
+                if os.path.exists(annotation_path + lbl_file) and os.path.exists(pose_path + lbl_file):
+                    img_filenames.append(str(file))
+                    landmark_filenames.append(lbl_file)
+                    poses_filenames.append(pose_file)
+
+        return np.array(img_filenames), np.array(landmark_filenames), np.array(poses_filenames)
+
+    def _get_batch_sample(self, batch_index, img_filenames, landmark_filenames, pose_filenames):
+        """
+        :param batch_index:
+        :param img_filenames:
+        :param landmark_filenames:
+        :param pose_filenames:
+        :return:
+        """
+
+        '''create batch data and normalize images'''
+        batch_img = img_filenames[
+                    batch_index * LearningConfig.batch_size:(batch_index + 1) * LearningConfig.batch_size]
+        batch_lnd = landmark_filenames[
+                    batch_index * LearningConfig.batch_size:(batch_index + 1) * LearningConfig.batch_size]
+        batch_pose = pose_filenames[
+                     batch_index * LearningConfig.batch_size:(batch_index + 1) * LearningConfig.batch_size]
+        '''create img and annotations'''
+        img_batch = np.array([imread(self.img_path + file_name) for file_name in batch_img]) / 255.0
+        lnd_batch = np.array([self._load_and_normalize(self.annotation_path + file_name) for file_name in batch_lnd])
+        pose_batch = np.array([load(self.pose_path + file_name) for file_name in batch_pose])
+
+        return img_batch, lnd_batch, pose_batch
+
+    def _load_and_normalize(self, point_path):
+        """
+        :param point_path:
+        :return:
+        """
+
+        annotation = load(point_path)
+        '''normalize landmarks'''
+        width = InputDataSize.image_input_size
+        height = InputDataSize.image_input_size
+        x_center = width / 2
+        y_center = height / 2
+        annotation_norm = []
+        for p in range(0, len(annotation), 2):
+            annotation_norm.append((x_center - annotation[p]) / width)
+            annotation_norm.append((y_center - annotation[p + 1]) / height)
+        return annotation_norm