sanket kheni commited on
Commit
e20f0de
1 Parent(s): 635c654
.gitattributes CHANGED
@@ -1,34 +1,27 @@
1
  *.7z filter=lfs diff=lfs merge=lfs -text
2
  *.arrow filter=lfs diff=lfs merge=lfs -text
3
  *.bin filter=lfs diff=lfs merge=lfs -text
 
4
  *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
  *.ftz filter=lfs diff=lfs merge=lfs -text
7
  *.gz filter=lfs diff=lfs merge=lfs -text
8
  *.h5 filter=lfs diff=lfs merge=lfs -text
9
  *.joblib filter=lfs diff=lfs merge=lfs -text
10
  *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
  *.model filter=lfs diff=lfs merge=lfs -text
13
  *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
  *.onnx filter=lfs diff=lfs merge=lfs -text
17
  *.ot filter=lfs diff=lfs merge=lfs -text
18
  *.parquet filter=lfs diff=lfs merge=lfs -text
19
  *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
  *.pt filter=lfs diff=lfs merge=lfs -text
23
  *.pth filter=lfs diff=lfs merge=lfs -text
24
  *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
  saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
  *.tar.* filter=lfs diff=lfs merge=lfs -text
28
  *.tflite filter=lfs diff=lfs merge=lfs -text
29
  *.tgz filter=lfs diff=lfs merge=lfs -text
30
- *.wasm filter=lfs diff=lfs merge=lfs -text
31
  *.xz filter=lfs diff=lfs merge=lfs -text
32
  *.zip filter=lfs diff=lfs merge=lfs -text
33
- *.zst filter=lfs diff=lfs merge=lfs -text
34
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
1
  *.7z filter=lfs diff=lfs merge=lfs -text
2
  *.arrow filter=lfs diff=lfs merge=lfs -text
3
  *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bin.* filter=lfs diff=lfs merge=lfs -text
5
  *.bz2 filter=lfs diff=lfs merge=lfs -text
 
6
  *.ftz filter=lfs diff=lfs merge=lfs -text
7
  *.gz filter=lfs diff=lfs merge=lfs -text
8
  *.h5 filter=lfs diff=lfs merge=lfs -text
9
  *.joblib filter=lfs diff=lfs merge=lfs -text
10
  *.lfs.* filter=lfs diff=lfs merge=lfs -text
 
11
  *.model filter=lfs diff=lfs merge=lfs -text
12
  *.msgpack filter=lfs diff=lfs merge=lfs -text
 
 
13
  *.onnx filter=lfs diff=lfs merge=lfs -text
14
  *.ot filter=lfs diff=lfs merge=lfs -text
15
  *.parquet filter=lfs diff=lfs merge=lfs -text
16
  *.pb filter=lfs diff=lfs merge=lfs -text
 
 
17
  *.pt filter=lfs diff=lfs merge=lfs -text
18
  *.pth filter=lfs diff=lfs merge=lfs -text
19
  *.rar filter=lfs diff=lfs merge=lfs -text
 
20
  saved_model/**/* filter=lfs diff=lfs merge=lfs -text
21
  *.tar.* filter=lfs diff=lfs merge=lfs -text
22
  *.tflite filter=lfs diff=lfs merge=lfs -text
23
  *.tgz filter=lfs diff=lfs merge=lfs -text
 
24
  *.xz filter=lfs diff=lfs merge=lfs -text
25
  *.zip filter=lfs diff=lfs merge=lfs -text
26
+ *.zstandard filter=lfs diff=lfs merge=lfs -text
27
  *tfevents* filter=lfs diff=lfs merge=lfs -text
.gitignore.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ g_model
2
+ flagged
3
+ arcface_model
4
+ retina_model
.idea/.gitignore ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # Default ignored files
2
+ /shelf/
3
+ /workspace.xml
.idea/AFFA-face-swap.iml ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <module type="PYTHON_MODULE" version="4">
3
+ <component name="NewModuleRootManager">
4
+ <content url="file://$MODULE_DIR$" />
5
+ <orderEntry type="jdk" jdkName="Python 3.8 (py38)" jdkType="Python SDK" />
6
+ <orderEntry type="sourceFolder" forTests="false" />
7
+ </component>
8
+ <component name="PyDocumentationSettings">
9
+ <option name="format" value="PLAIN" />
10
+ <option name="myDocStringFormat" value="Plain" />
11
+ </component>
12
+ <component name="TestRunnerService">
13
+ <option name="PROJECT_TEST_RUNNER" value="pytest" />
14
+ </component>
15
+ </module>
.idea/inspectionProfiles/Project_Default.xml ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <component name="InspectionProjectProfileManager">
2
+ <profile version="1.0">
3
+ <option name="myName" value="Project Default" />
4
+ <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
5
+ <option name="ignoredPackages">
6
+ <value>
7
+ <list size="3">
8
+ <item index="0" class="java.lang.String" itemvalue="ipython" />
9
+ <item index="1" class="java.lang.String" itemvalue="Cython" />
10
+ <item index="2" class="java.lang.String" itemvalue="tensorflow-gpu" />
11
+ </list>
12
+ </value>
13
+ </option>
14
+ </inspection_tool>
15
+ <inspection_tool class="PyPep8Inspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
16
+ <option name="ignoredErrors">
17
+ <list>
18
+ <option value="E402" />
19
+ </list>
20
+ </option>
21
+ </inspection_tool>
22
+ <inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
23
+ <option name="ignoredErrors">
24
+ <list>
25
+ <option value="N806" />
26
+ <option value="N812" />
27
+ </list>
28
+ </option>
29
+ </inspection_tool>
30
+ <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
31
+ <option name="ignoredIdentifiers">
32
+ <list>
33
+ <option value="torch.backends.cudnn" />
34
+ </list>
35
+ </option>
36
+ </inspection_tool>
37
+ </profile>
38
+ </component>
.idea/inspectionProfiles/profiles_settings.xml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ <component name="InspectionProjectProfileManager">
2
+ <settings>
3
+ <option name="USE_PROJECT_PROFILE" value="false" />
4
+ <version value="1.0" />
5
+ </settings>
6
+ </component>
.idea/misc.xml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (py38)" project-jdk-type="Python SDK" />
4
+ </project>
.idea/modules.xml ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="ProjectModuleManager">
4
+ <modules>
5
+ <module fileurl="file://$PROJECT_DIR$/.idea/AFFA-face-swap.iml" filepath="$PROJECT_DIR$/.idea/AFFA-face-swap.iml" />
6
+ </modules>
7
+ </component>
8
+ </project>
.idea/vcs.xml ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="VcsDirectoryMappings">
4
+ <mapping directory="$PROJECT_DIR$" vcs="Git" />
5
+ <mapping directory="$PROJECT_DIR$/arcface_model" vcs="Git" />
6
+ <mapping directory="$PROJECT_DIR$/g_model" vcs="Git" />
7
+ <mapping directory="$PROJECT_DIR$/retina_model" vcs="Git" />
8
+ </component>
9
+ </project>
README.md CHANGED
@@ -1,12 +1,46 @@
1
  ---
2
- title: Powerswp
3
- emoji: 🌖
4
- colorFrom: red
5
- colorTo: yellow
6
  sdk: gradio
7
- sdk_version: 3.14.0
8
  app_file: app.py
9
  pinned: false
 
10
  ---
11
 
12
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: Face Swap
3
+ emoji: 🧙🧙🧙🧙🧙🧙🧙🧙
4
+ colorFrom: purple
5
+ colorTo: green
6
  sdk: gradio
 
7
  app_file: app.py
8
  pinned: false
9
+ license: cc-by-nc-sa-4.0
10
  ---
11
 
12
+ # Configuration
13
+
14
+ `title`: _string_
15
+ Display title for the Space
16
+
17
+ `emoji`: _string_
18
+ Space emoji (emoji-only character allowed)
19
+
20
+ `colorFrom`: _string_
21
+ Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
22
+
23
+ `colorTo`: _string_
24
+ Color for Thumbnail gradient (red, yellow, green, blue, indigo, purple, pink, gray)
25
+
26
+ `sdk`: _string_
27
+ Can be either `gradio`, `streamlit`, or `static`
28
+
29
+ `sdk_version` : _string_
30
+ Only applicable for `streamlit` SDK.
31
+ See [doc](https://hf.co/docs/hub/spaces) for more info on supported versions.
32
+
33
+ `app_file`: _string_
34
+ Path to your main application file (which contains either `gradio` or `streamlit` Python code, or `static` html code).
35
+ Path is relative to the root of the repository.
36
+
37
+ `models`: _List[string]_
38
+ HF model IDs (like "gpt2" or "deepset/roberta-base-squad2") used in the Space.
39
+ Will be parsed automatically from your code if not specified here.
40
+
41
+ `datasets`: _List[string]_
42
+ HF dataset IDs (like "common_voice" or "oscar-corpus/OSCAR-2109") used in the Space.
43
+ Will be parsed automatically from your code if not specified here.
44
+
45
+ `pinned`: _boolean_
46
+ Whether the Space stays on top of your list.
app.py ADDED
@@ -0,0 +1,216 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio
2
+ from huggingface_hub import Repository
3
+ import os
4
+
5
+ from utils.utils import norm_crop, estimate_norm, inverse_estimate_norm, transform_landmark_points, get_lm
6
+ from networks.layers import AdaIN, AdaptiveAttention
7
+ from tensorflow_addons.layers import InstanceNormalization
8
+ import numpy as np
9
+ import cv2
10
+ from scipy.ndimage import gaussian_filter
11
+
12
+ from tensorflow.keras.models import load_model
13
+ from options.swap_options import SwapOptions
14
+
15
+ # .
16
+ token = os.environ['model_fetch']
17
+
18
+ opt = SwapOptions().parse()
19
+
20
+ retina_repo = Repository(local_dir="retina_model", clone_from="felixrosberg/retinaface_resnet50",
21
+ private=True, use_auth_token=token, git_user="felixrosberg")
22
+
23
+ from retina_model.models import *
24
+
25
+ RetinaFace = load_model("retina_model/retinaface_res50.h5",
26
+ custom_objects={"FPN": FPN,
27
+ "SSH": SSH,
28
+ "BboxHead": BboxHead,
29
+ "LandmarkHead": LandmarkHead,
30
+ "ClassHead": ClassHead})
31
+
32
+ arc_repo = Repository(local_dir="arcface_model", clone_from="felixrosberg/arcface_tf",
33
+ private=True, use_auth_token=token)
34
+ ArcFace = load_model("arcface_model/arc_res50.h5")
35
+ ArcFaceE = load_model("arcface_model/arc_res50e.h5")
36
+
37
+ g_repo = Repository(local_dir="g_model_c_hq", clone_from="felixrosberg/affa_config_c_hq",
38
+ private=True, use_auth_token=token)
39
+ G = load_model("g_model_c_hq/generator_t_28.h5", custom_objects={"AdaIN": AdaIN,
40
+ "AdaptiveAttention": AdaptiveAttention,
41
+ "InstanceNormalization": InstanceNormalization})
42
+
43
+ r_repo = Repository(local_dir="reconstruction_attack", clone_from="felixrosberg/reconstruction_attack",
44
+ private=True, use_auth_token=token)
45
+ R = load_model("reconstruction_attack/reconstructor_42.h5", custom_objects={"AdaIN": AdaIN,
46
+ "AdaptiveAttention": AdaptiveAttention,
47
+ "InstanceNormalization": InstanceNormalization})
48
+
49
+ permuter_repo = Repository(local_dir="identity_permuter", clone_from="felixrosberg/identitypermuter",
50
+ private=True, use_auth_token=token, git_user="felixrosberg")
51
+
52
+ from identity_permuter.id_permuter import identity_permuter
53
+
54
+ IDP = identity_permuter(emb_size=32, min_arg=False)
55
+ IDP.load_weights("identity_permuter/id_permuter.h5")
56
+
57
+ blend_mask_base = np.zeros(shape=(256, 256, 1))
58
+ blend_mask_base[80:244, 32:224] = 1
59
+ blend_mask_base = gaussian_filter(blend_mask_base, sigma=7)
60
+
61
+
62
+ def run_inference(target, source, slider, adv_slider, settings):
63
+ try:
64
+ source = np.array(source)
65
+ target = np.array(target)
66
+
67
+ # Prepare to load video
68
+ if "anonymize" not in settings:
69
+ source_a = RetinaFace(np.expand_dims(source, axis=0)).numpy()[0]
70
+ source_h, source_w, _ = source.shape
71
+ source_lm = get_lm(source_a, source_w, source_h)
72
+ source_aligned = norm_crop(source, source_lm, image_size=256)
73
+ source_z = ArcFace.predict(np.expand_dims(tf.image.resize(source_aligned, [112, 112]) / 255.0, axis=0))
74
+ else:
75
+ source_z = None
76
+
77
+ # read frame
78
+ im = target
79
+ im_h, im_w, _ = im.shape
80
+ im_shape = (im_w, im_h)
81
+
82
+ detection_scale = im_w // 640 if im_w > 640 else 1
83
+
84
+ faces = RetinaFace(np.expand_dims(cv2.resize(im,
85
+ (im_w // detection_scale,
86
+ im_h // detection_scale)), axis=0)).numpy()
87
+
88
+ total_img = im / 255.0
89
+ for annotation in faces:
90
+ lm_align = np.array([[annotation[4] * im_w, annotation[5] * im_h],
91
+ [annotation[6] * im_w, annotation[7] * im_h],
92
+ [annotation[8] * im_w, annotation[9] * im_h],
93
+ [annotation[10] * im_w, annotation[11] * im_h],
94
+ [annotation[12] * im_w, annotation[13] * im_h]],
95
+ dtype=np.float32)
96
+
97
+ # align the detected face
98
+ M, pose_index = estimate_norm(lm_align, 256, "arcface", shrink_factor=1.0)
99
+ im_aligned = (cv2.warpAffine(im, M, (256, 256), borderValue=0.0) - 127.5) / 127.5
100
+
101
+ if "adversarial defense" in settings:
102
+ eps = adv_slider / 200
103
+ X = tf.convert_to_tensor(np.expand_dims(im_aligned, axis=0))
104
+ with tf.GradientTape() as tape:
105
+ tape.watch(X)
106
+
107
+ X_z = ArcFaceE(tf.image.resize(X * 0.5 + 0.5, [112, 112]))
108
+ output = R([X, X_z])
109
+
110
+ loss = tf.reduce_mean(tf.abs(0 - output))
111
+
112
+ gradient = tf.sign(tape.gradient(loss, X))
113
+
114
+ adv_x = X + eps * gradient
115
+ im_aligned = tf.clip_by_value(adv_x, -1, 1)[0]
116
+
117
+ if "anonymize" in settings and "reconstruction attack" not in settings:
118
+ """source_z = ArcFace.predict(np.expand_dims(tf.image.resize(im_aligned, [112, 112]) / 255.0, axis=0))
119
+ anon_ratio = int(512 * (slider / 100))
120
+ anon_vector = np.ones(shape=(1, 512))
121
+ anon_vector[:, :anon_ratio] = -1
122
+ np.random.shuffle(anon_vector)
123
+ source_z *= anon_vector"""
124
+
125
+ slider_weight = slider / 100
126
+
127
+ target_z = ArcFace.predict(np.expand_dims(tf.image.resize(im_aligned, [112, 112]) * 0.5 + 0.5, axis=0))
128
+ source_z = IDP.predict(target_z)
129
+
130
+ source_z = slider_weight * source_z + (1 - slider_weight) * target_z
131
+
132
+ if "reconstruction attack" in settings:
133
+ source_z = ArcFaceE.predict(np.expand_dims(tf.image.resize(im_aligned, [112, 112]) * 0.5 + 0.5, axis=0))
134
+
135
+ # face swap
136
+ if "reconstruction attack" not in settings:
137
+ changed_face_cage = G.predict([np.expand_dims(im_aligned, axis=0),
138
+ source_z])
139
+ changed_face = changed_face_cage[0] * 0.5 + 0.5
140
+
141
+ # get inverse transformation landmarks
142
+ transformed_lmk = transform_landmark_points(M, lm_align)
143
+
144
+ # warp image back
145
+ iM, _ = inverse_estimate_norm(lm_align, transformed_lmk, 256, "arcface", shrink_factor=1.0)
146
+ iim_aligned = cv2.warpAffine(changed_face, iM, im_shape, borderValue=0.0)
147
+
148
+ # blend swapped face with target image
149
+ blend_mask = cv2.warpAffine(blend_mask_base, iM, im_shape, borderValue=0.0)
150
+ blend_mask = np.expand_dims(blend_mask, axis=-1)
151
+ total_img = (iim_aligned * blend_mask + total_img * (1 - blend_mask))
152
+ else:
153
+ changed_face_cage = R.predict([np.expand_dims(im_aligned, axis=0),
154
+ source_z])
155
+ changed_face = changed_face_cage[0] * 0.5 + 0.5
156
+
157
+ # get inverse transformation landmarks
158
+ transformed_lmk = transform_landmark_points(M, lm_align)
159
+
160
+ # warp image back
161
+ iM, _ = inverse_estimate_norm(lm_align, transformed_lmk, 256, "arcface", shrink_factor=1.0)
162
+ iim_aligned = cv2.warpAffine(changed_face, iM, im_shape, borderValue=0.0)
163
+
164
+ # blend swapped face with target image
165
+ blend_mask = cv2.warpAffine(blend_mask_base, iM, im_shape, borderValue=0.0)
166
+ blend_mask = np.expand_dims(blend_mask, axis=-1)
167
+ total_img = (iim_aligned * blend_mask + total_img * (1 - blend_mask))
168
+
169
+ if "compare" in settings:
170
+ total_img = np.concatenate((im / 255.0, total_img), axis=1)
171
+
172
+ total_img = np.clip(total_img, 0, 1)
173
+ total_img *= 255.0
174
+ total_img = total_img.astype('uint8')
175
+
176
+ return total_img
177
+ except Exception as e:
178
+ print(e)
179
+ return None
180
+
181
+
182
+ description = "Performs subject agnostic identity transfer from a source face to all target faces. \n\n" \
183
+ "Implementation and demo of FaceDancer, accepted to WACV 2023. \n\n" \
184
+ "Pre-print: https://arxiv.org/abs/2210.10473 \n\n" \
185
+ "Code: https://github.com/felixrosberg/FaceDancer \n\n" \
186
+ "\n\n" \
187
+ "Options:\n\n" \
188
+ "-Compare returns the target image concatenated with the results.\n\n" \
189
+ "-Anonymize will ignore the source image and perform an identity permutation of target faces.\n\n" \
190
+ "-Reconstruction attack will attempt to invert the face swap or the anonymization.\n\n" \
191
+ "-Adversarial defense will add a permutation noise that disrupts the reconstruction attack.\n\n" \
192
+ "NOTE: There is no guarantees with the anonymization process currently.\n\n" \
193
+ "NOTE: source image with too high resolution may not work properly!"
194
+ examples = [["assets/rick.jpg", "assets/musk.jpg", 100, 10, ["compare"]],
195
+ ["assets/musk.jpg", "assets/musk.jpg", 100, 10, ["anonymize"]]]
196
+ article = """
197
+ Demo is based of recent research from my Ph.D work. Results expects to be published in the coming months.
198
+ """
199
+
200
+ iface = gradio.Interface(run_inference,
201
+ [gradio.inputs.Image(shape=None, label='Target'),
202
+ gradio.inputs.Image(shape=None, label='Source'),
203
+ gradio.inputs.Slider(0, 100, default=100, label="Anonymization ratio (%)"),
204
+ gradio.inputs.Slider(0, 100, default=100, label="Adversarial defense ratio (%)"),
205
+ gradio.inputs.CheckboxGroup(["compare",
206
+ "anonymize",
207
+ "reconstruction attack",
208
+ "adversarial defense"],
209
+ label='Options')],
210
+ gradio.outputs.Image(),
211
+ title="Face Swap",
212
+ description=description,
213
+ examples=examples,
214
+ article=article,
215
+ layout="vertical")
216
+ iface.launch()
assets/girl_0.png ADDED
assets/girl_1.png ADDED
assets/musk.jpg ADDED
assets/rick.jpg ADDED
networks/__pycache__/layers.cpython-37.pyc ADDED
Binary file (69.1 kB). View file
 
networks/__pycache__/layers.cpython-38.pyc ADDED
Binary file (2.12 kB). View file
 
networks/layers.py ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import tensorflow as tf
2
+ from tensorflow.keras.layers import Layer, Dense
3
+
4
+
5
+ def sin_activation(x, omega=30):
6
+ return tf.math.sin(omega * x)
7
+
8
+
9
+ class AdaIN(Layer):
10
+ def __init__(self, **kwargs):
11
+ super(AdaIN, self).__init__(**kwargs)
12
+
13
+ def build(self, input_shapes):
14
+ x_shape = input_shapes[0]
15
+ w_shape = input_shapes[1]
16
+
17
+ self.w_channels = w_shape[-1]
18
+ self.x_channels = x_shape[-1]
19
+
20
+ self.dense_1 = Dense(self.x_channels)
21
+ self.dense_2 = Dense(self.x_channels)
22
+
23
+ def call(self, inputs):
24
+ x, w = inputs
25
+ ys = tf.reshape(self.dense_1(w), (-1, 1, 1, self.x_channels))
26
+ yb = tf.reshape(self.dense_2(w), (-1, 1, 1, self.x_channels))
27
+ return ys * x + yb
28
+
29
+ def get_config(self):
30
+ config = {
31
+ #'w_channels': self.w_channels,
32
+ #'x_channels': self.x_channels
33
+ }
34
+ base_config = super(AdaIN, self).get_config()
35
+ return dict(list(base_config.items()) + list(config.items()))
36
+
37
+
38
+ class AdaptiveAttention(Layer):
39
+
40
+ def __init__(self, **kwargs):
41
+ super(AdaptiveAttention, self).__init__(**kwargs)
42
+
43
+ def call(self, inputs):
44
+ m, a, i = inputs
45
+ return (1 - m) * a + m * i
46
+
47
+ def get_config(self):
48
+ base_config = super(AdaptiveAttention, self).get_config()
49
+ return base_config
options/__pycache__/swap_options.cpython-37.pyc ADDED
Binary file (6.21 kB). View file
 
options/__pycache__/swap_options.cpython-38.pyc ADDED
Binary file (1.65 kB). View file
 
options/swap_options.py ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+
3
+
4
+ class SwapOptions():
5
+ def __init__(self):
6
+ self.parser = argparse.ArgumentParser()
7
+ self.initialized = False
8
+
9
+ def initialize(self):
10
+ # paths (data, models, etc...)
11
+ self.parser.add_argument('--arcface_path', type=str,
12
+ default="arcface_model/arcface/arc_res50.h5",
13
+ help='path to arcface model. Used to extract identity from source.')
14
+
15
+ # Video/Image necessary models
16
+ self.parser.add_argument('--retina_path', type=str,
17
+ default="retinaface/retinaface_res50.h5",
18
+ help='path to retinaface model.')
19
+ self.parser.add_argument('--compare', type=bool,
20
+ default=True,
21
+ help='If true, concatenates the frame with the manipulated frame')
22
+
23
+ self.parser.add_argument('--load', type=int,
24
+ default=30,
25
+ help='int of number to load checkpoint weights.')
26
+ self.parser.add_argument('--device_id', type=int, default=0,
27
+ help='which device to use')
28
+
29
+ # logging and checkpointing
30
+ self.parser.add_argument('--log_dir', type=str, default='logs/runs/',
31
+ help='logging directory')
32
+ self.parser.add_argument('--log_name', type=str, default='affa_f',
33
+ help='name of the run, change this to track several experiments')
34
+
35
+ self.parser.add_argument('--chkp_dir', type=str, default='checkpoints/',
36
+ help='checkpoint directory (will use same name as log_name!)')
37
+ self.initialized = True
38
+
39
+ def parse(self):
40
+ if not self.initialized:
41
+ self.initialize()
42
+ self.opt = self.parser.parse_args()
43
+ return self.opt
requirements.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ tensorflow
2
+ tensorflow-addons
3
+ opencv-python-headless
4
+ scipy
5
+ pillow
6
+ scikit-image
7
+ huggingface_hub
utils/__pycache__/utils.cpython-38.pyc ADDED
Binary file (11.6 kB). View file
 
utils/utils.py ADDED
@@ -0,0 +1,377 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from tensorflow.keras.models import model_from_json
3
+ from networks.layers import AdaIN, AdaptiveAttention
4
+ import tensorflow as tf
5
+
6
+ import numpy as np
7
+ import cv2
8
+ import math
9
+ from skimage import transform as trans
10
+ from scipy.signal import convolve2d
11
+ from skimage.color import rgb2yuv, yuv2rgb
12
+
13
+ from PIL import Image
14
+
15
+
16
+ def save_model_internal(model, path, name, num):
17
+ json_model = model.to_json()
18
+ with open(path + name + '.json', "w") as json_file:
19
+ json_file.write(json_model)
20
+
21
+ model.save_weights(path + name + '_' + str(num) + '.h5')
22
+
23
+
24
+ def load_model_internal(path, name, num):
25
+ with open(path + name + '.json', 'r') as json_file:
26
+ model_dict = json_file.read()
27
+
28
+ mod = model_from_json(model_dict, custom_objects={'AdaIN': AdaIN, 'AdaptiveAttention': AdaptiveAttention})
29
+ mod.load_weights(path + name + '_' + str(num) + '.h5')
30
+
31
+ return mod
32
+
33
+
34
+ def save_training_meta(state_dict, path, num):
35
+ with open(path + str(num) + '.json', 'w') as json_file:
36
+ json.dump(state_dict, json_file, indent=2)
37
+
38
+
39
+ def load_training_meta(path, num):
40
+ with open(path + str(num) + '.json', 'r') as json_file:
41
+ state_dict = json.load(json_file)
42
+ return state_dict
43
+
44
+
45
+ def log_info(sw, results_dict, iteration):
46
+ with sw.as_default():
47
+ for key in results_dict.keys():
48
+ tf.summary.scalar(key, results_dict[key], step=iteration)
49
+
50
+
51
+ src1 = np.array([[51.642, 50.115], [57.617, 49.990], [35.740, 69.007],
52
+ [51.157, 89.050], [57.025, 89.702]],
53
+ dtype=np.float32)
54
+ # <--left
55
+ src2 = np.array([[45.031, 50.118], [65.568, 50.872], [39.677, 68.111],
56
+ [45.177, 86.190], [64.246, 86.758]],
57
+ dtype=np.float32)
58
+
59
+ # ---frontal
60
+ src3 = np.array([[39.730, 51.138], [72.270, 51.138], [56.000, 68.493],
61
+ [42.463, 87.010], [69.537, 87.010]],
62
+ dtype=np.float32)
63
+
64
+ # -->right
65
+ src4 = np.array([[46.845, 50.872], [67.382, 50.118], [72.737, 68.111],
66
+ [48.167, 86.758], [67.236, 86.190]],
67
+ dtype=np.float32)
68
+
69
+ # -->right profile
70
+ src5 = np.array([[54.796, 49.990], [60.771, 50.115], [76.673, 69.007],
71
+ [55.388, 89.702], [61.257, 89.050]],
72
+ dtype=np.float32)
73
+
74
+ src = np.array([src1, src2, src3, src4, src5])
75
+ src_map = {112: src, 224: src * 2}
76
+
77
+ # Left eye, right eye, nose, left mouth, right mouth
78
+ arcface_src = np.array(
79
+ [[38.2946, 51.6963], [73.5318, 51.5014], [56.0252, 71.7366],
80
+ [41.5493, 92.3655], [70.7299, 92.2041]],
81
+ dtype=np.float32)
82
+
83
+ arcface_src = np.expand_dims(arcface_src, axis=0)
84
+
85
+
86
+ def extract_face(img, bb, absolute_center, mode='arcface', extention_rate=0.05, debug=False):
87
+ """Extract face from image given a bounding box"""
88
+ # bbox
89
+ x1, y1, x2, y2 = bb + 60
90
+ adjusted_absolute_center = (absolute_center[0] + 60, absolute_center[1] + 60)
91
+ if debug:
92
+ print(bb + 60)
93
+ x1, y1, x2, y2 = bb
94
+ cv2.rectangle(img, (x1, y1), (x2, y2), (0, 255, 0), 3)
95
+ cv2.circle(img, absolute_center, 1, (255, 0, 255), 2)
96
+ Image.fromarray(img).show()
97
+ x1, y1, x2, y2 = bb + 60
98
+ # Pad image in case face is out of frame
99
+ padded_img = np.zeros(shape=(248, 248, 3), dtype=np.uint8)
100
+ padded_img[60:-60, 60:-60, :] = img
101
+
102
+ if debug:
103
+ cv2.rectangle(padded_img, (x1, y1), (x2, y2), (0, 255, 255), 3)
104
+ cv2.circle(padded_img, adjusted_absolute_center, 1, (255, 255, 255), 2)
105
+ Image.fromarray(padded_img).show()
106
+
107
+ y_len = abs(y1 - y2)
108
+ x_len = abs(x1 - x2)
109
+
110
+ new_len = (y_len + x_len) // 2
111
+
112
+ extension = int(new_len * extention_rate)
113
+
114
+ x_adjust = (x_len - new_len) // 2
115
+ y_adjust = (y_len - new_len) // 2
116
+
117
+ x_1_adjusted = x1 + x_adjust - extension
118
+ x_2_adjusted = x2 - x_adjust + extension
119
+
120
+ if mode == 'arcface':
121
+ y_1_adjusted = y1 - extension
122
+ y_2_adjusted = y2 - 2 * y_adjust + extension
123
+ else:
124
+ y_1_adjusted = y1 + 2 * y_adjust - extension
125
+ y_2_adjusted = y2 + extension
126
+
127
+ move_x = adjusted_absolute_center[0] - (x_1_adjusted + x_2_adjusted) // 2
128
+ move_y = adjusted_absolute_center[1] - (y_1_adjusted + y_2_adjusted) // 2
129
+
130
+ x_1_adjusted = x_1_adjusted + move_x
131
+ x_2_adjusted = x_2_adjusted + move_x
132
+ y_1_adjusted = y_1_adjusted + move_y
133
+ y_2_adjusted = y_2_adjusted + move_y
134
+
135
+ # print(y_1_adjusted, y_2_adjusted, x_1_adjusted, x_2_adjusted)
136
+
137
+ return padded_img[y_1_adjusted:y_2_adjusted, x_1_adjusted:x_2_adjusted]
138
+
139
+
140
+ def distance(a, b):
141
+ return np.sqrt((a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2)
142
+
143
+
144
+ def euclidean_distance(a, b):
145
+ x1 = a[0]; y1 = a[1]
146
+ x2 = b[0]; y2 = b[1]
147
+ return np.sqrt(((x2 - x1) * (x2 - x1)) + ((y2 - y1) * (y2 - y1)))
148
+
149
+
150
+ def align_face(img, landmarks, debug=False):
151
+ nose, right_eye, left_eye = landmarks
152
+
153
+ left_eye_x = left_eye[0]
154
+ left_eye_y = left_eye[1]
155
+
156
+ right_eye_x = right_eye[0]
157
+ right_eye_y = right_eye[1]
158
+
159
+ center_eye = ((left_eye[0] + right_eye[0]) // 2, (left_eye[1] + right_eye[1]) // 2)
160
+
161
+ if left_eye_y < right_eye_y:
162
+ point_3rd = (right_eye_x, left_eye_y)
163
+ direction = -1
164
+ else:
165
+ point_3rd = (left_eye_x, right_eye_y)
166
+ direction = 1
167
+
168
+ if debug:
169
+ cv2.circle(img, point_3rd, 1, (255, 0, 0), 1)
170
+ cv2.circle(img, center_eye, 1, (255, 0, 0), 1)
171
+
172
+ cv2.line(img, right_eye, left_eye, (0, 0, 0), 1)
173
+ cv2.line(img, left_eye, point_3rd, (0, 0, 0), 1)
174
+ cv2.line(img, right_eye, point_3rd, (0, 0, 0), 1)
175
+
176
+ a = euclidean_distance(left_eye, point_3rd)
177
+ b = euclidean_distance(right_eye, left_eye)
178
+ c = euclidean_distance(right_eye, point_3rd)
179
+
180
+ cos_a = (b * b + c * c - a * a) / (2 * b * c)
181
+
182
+ angle = np.arccos(cos_a)
183
+
184
+ angle = (angle * 180) / np.pi
185
+
186
+ if direction == -1:
187
+ angle = 90 - angle
188
+ ang = math.radians(direction * angle)
189
+ else:
190
+ ang = math.radians(direction * angle)
191
+ angle = 0 - angle
192
+
193
+ M = cv2.getRotationMatrix2D((64, 64), angle, 1)
194
+ new_img = cv2.warpAffine(img, M, (128, 128),
195
+ flags=cv2.INTER_CUBIC)
196
+
197
+ rotated_nose = (int((nose[0] - 64) * np.cos(ang) - (nose[1] - 64) * np.sin(ang) + 64),
198
+ int((nose[0] - 64) * np.sin(ang) + (nose[1] - 64) * np.cos(ang) + 64))
199
+
200
+ rotated_center_eye = (int((center_eye[0] - 64) * np.cos(ang) - (center_eye[1] - 64) * np.sin(ang) + 64),
201
+ int((center_eye[0] - 64) * np.sin(ang) + (center_eye[1] - 64) * np.cos(ang) + 64))
202
+
203
+ abolute_center = (rotated_center_eye[0], (rotated_nose[1] + rotated_center_eye[1]) // 2)
204
+
205
+ if debug:
206
+ cv2.circle(new_img, rotated_nose, 1, (0, 0, 255), 1)
207
+ cv2.circle(new_img, rotated_center_eye, 1, (0, 0, 255), 1)
208
+ cv2.circle(new_img, abolute_center, 1, (0, 0, 255), 1)
209
+
210
+ return new_img, abolute_center
211
+
212
+
213
+ def estimate_norm(lmk, image_size=112, mode='arcface', shrink_factor=1.0):
214
+ assert lmk.shape == (5, 2)
215
+ tform = trans.SimilarityTransform()
216
+ lmk_tran = np.insert(lmk, 2, values=np.ones(5), axis=1)
217
+ min_M = []
218
+ min_index = []
219
+ min_error = float('inf')
220
+ src_factor = image_size / 112
221
+ if mode == 'arcface':
222
+ src = arcface_src * shrink_factor + (1 - shrink_factor) * 56
223
+ src = src * src_factor
224
+ else:
225
+ src = src_map[image_size] * src_factor
226
+ for i in np.arange(src.shape[0]):
227
+ tform.estimate(lmk, src[i])
228
+ M = tform.params[0:2, :]
229
+ results = np.dot(M, lmk_tran.T)
230
+ results = results.T
231
+ error = np.sum(np.sqrt(np.sum((results - src[i])**2, axis=1)))
232
+ # print(error)
233
+ if error < min_error:
234
+ min_error = error
235
+ min_M = M
236
+ min_index = i
237
+ return min_M, min_index
238
+
239
+
240
+ def inverse_estimate_norm(lmk, t_lmk, image_size=112, mode='arcface', shrink_factor=1.0):
241
+ assert lmk.shape == (5, 2)
242
+ tform = trans.SimilarityTransform()
243
+ lmk_tran = np.insert(lmk, 2, values=np.ones(5), axis=1)
244
+ min_M = []
245
+ min_index = []
246
+ min_error = float('inf')
247
+ src_factor = image_size / 112
248
+ if mode == 'arcface':
249
+ src = arcface_src * shrink_factor + (1 - shrink_factor) * 56
250
+ src = src * src_factor
251
+ else:
252
+ src = src_map[image_size] * src_factor
253
+ for i in np.arange(src.shape[0]):
254
+ tform.estimate(t_lmk, lmk)
255
+ M = tform.params[0:2, :]
256
+ results = np.dot(M, lmk_tran.T)
257
+ results = results.T
258
+ error = np.sum(np.sqrt(np.sum((results - src[i])**2, axis=1)))
259
+ # print(error)
260
+ if error < min_error:
261
+ min_error = error
262
+ min_M = M
263
+ min_index = i
264
+ return min_M, min_index
265
+
266
+
267
+ def norm_crop(img, landmark, image_size=112, mode='arcface', shrink_factor=1.0):
268
+ """
269
+ Align and crop the image based of the facial landmarks in the image. The alignment is done with
270
+ a similarity transformation based of source coordinates.
271
+ :param img: Image to transform.
272
+ :param landmark: Five landmark coordinates in the image.
273
+ :param image_size: Desired output size after transformation.
274
+ :param mode: 'arcface' aligns the face for the use of Arcface facial recognition model. Useful for
275
+ both facial recognition tasks and face swapping tasks.
276
+ :param shrink_factor: Shrink factor that shrinks the source landmark coordinates. This will include more border
277
+ information around the face. Useful when you want to include more background information when performing face swaps.
278
+ The lower the shrink factor the more of the face is included. Default value 1.0 will align the image to be ready
279
+ for the Arcface recognition model, but usually omits part of the chin. Value of 0.0 would transform all source points
280
+ to the middle of the image, probably rendering the alignment procedure useless.
281
+
282
+ If you process the image with a shrink factor of 0.85 and then want to extract the identity embedding with arcface,
283
+ you simply do a central crop of factor 0.85 to yield same cropped result as using shrink factor 1.0. This will
284
+ reduce the resolution, the recommendation is to processed images to output resolutions higher than 112 is using
285
+ Arcface. This will make sure no information is lost by resampling the image after central crop.
286
+ :return: Returns the transformed image.
287
+ """
288
+ M, pose_index = estimate_norm(landmark, image_size, mode, shrink_factor=shrink_factor)
289
+ warped = cv2.warpAffine(img, M, (image_size, image_size), borderValue=0.0)
290
+ return warped
291
+
292
+
293
+ def transform_landmark_points(M, points):
294
+ lmk_tran = np.insert(points, 2, values=np.ones(5), axis=1)
295
+ transformed_lmk = np.dot(M, lmk_tran.T)
296
+ transformed_lmk = transformed_lmk.T
297
+
298
+ return transformed_lmk
299
+
300
+
301
+ def multi_convolver(image, kernel, iterations):
302
+ if kernel == "Sharpen":
303
+ kernel = np.array([[0, -1, 0],
304
+ [-1, 5, -1],
305
+ [0, -1, 0]])
306
+ elif kernel == "Unsharp_mask":
307
+ kernel = np.array([[1, 4, 6, 4, 1],
308
+ [4, 16, 24, 16, 1],
309
+ [6, 24, -476, 24, 1],
310
+ [4, 16, 24, 16, 1],
311
+ [1, 4, 6, 4, 1]]) * (-1 / 256)
312
+ elif kernel == "Blur":
313
+ kernel = (1 / 16.0) * np.array([[1., 2., 1.],
314
+ [2., 4., 2.],
315
+ [1., 2., 1.]])
316
+ for i in range(iterations):
317
+ image = convolve2d(image, kernel, 'same', boundary='fill', fillvalue = 0)
318
+ return image
319
+
320
+
321
+ def convolve_rgb(image, kernel, iterations=1):
322
+ img_yuv = rgb2yuv(image)
323
+ img_yuv[:, :, 0] = multi_convolver(img_yuv[:, :, 0], kernel,
324
+ iterations)
325
+ final_image = yuv2rgb(img_yuv)
326
+
327
+ return final_image.astype('float32')
328
+
329
+
330
+ def generate_mask_from_landmarks(lms, im_size):
331
+ blend_mask_lm = np.zeros(shape=(im_size, im_size, 3), dtype='float32')
332
+
333
+ # EYES
334
+ blend_mask_lm = cv2.circle(blend_mask_lm,
335
+ (int(lms[0][0]), int(lms[0][1])), 12, (255, 255, 255), 30)
336
+ blend_mask_lm = cv2.circle(blend_mask_lm,
337
+ (int(lms[1][0]), int(lms[1][1])), 12, (255, 255, 255), 30)
338
+ blend_mask_lm = cv2.circle(blend_mask_lm,
339
+ (int((lms[0][0] + lms[1][0]) / 2), int((lms[0][1] + lms[1][1]) / 2)),
340
+ 16, (255, 255, 255), 65)
341
+
342
+ # NOSE
343
+ blend_mask_lm = cv2.circle(blend_mask_lm,
344
+ (int(lms[2][0]), int(lms[2][1])), 5, (255, 255, 255), 5)
345
+ blend_mask_lm = cv2.circle(blend_mask_lm,
346
+ (int((lms[0][0] + lms[1][0]) / 2), int(lms[2][1])), 16, (255, 255, 255), 100)
347
+
348
+ # MOUTH
349
+ blend_mask_lm = cv2.circle(blend_mask_lm,
350
+ (int(lms[3][0]), int(lms[3][1])), 6, (255, 255, 255), 30)
351
+ blend_mask_lm = cv2.circle(blend_mask_lm,
352
+ (int(lms[4][0]), int(lms[4][1])), 6, (255, 255, 255), 30)
353
+
354
+ blend_mask_lm = cv2.circle(blend_mask_lm,
355
+ (int((lms[3][0] + lms[4][0]) / 2), int((lms[3][1] + lms[4][1]) / 2)),
356
+ 16, (255, 255, 255), 40)
357
+ return blend_mask_lm
358
+
359
+
360
+ def display_distance_text(im, distance, lms, im_w, im_h, scale=2):
361
+ blended_insert = cv2.putText(im, str(distance)[:4],
362
+ (int(lms[4] * im_w * 0.5), int(lms[5] * im_h * 0.8)),
363
+ cv2.FONT_HERSHEY_SIMPLEX, scale * 0.5, (0.08, 0.16, 0.08), int(scale * 2))
364
+ blended_insert = cv2.putText(blended_insert, str(distance)[:4],
365
+ (int(lms[4] * im_w * 0.5), int(lms[5] * im_h * 0.8)),
366
+ cv2.FONT_HERSHEY_SIMPLEX, scale* 0.5, (0.3, 0.7, 0.32), int(scale * 1))
367
+ return blended_insert
368
+
369
+
370
+ def get_lm(annotation, im_w, im_h):
371
+ lm_align = np.array([[annotation[4] * im_w, annotation[5] * im_h],
372
+ [annotation[6] * im_w, annotation[7] * im_h],
373
+ [annotation[8] * im_w, annotation[9] * im_h],
374
+ [annotation[10] * im_w, annotation[11] * im_h],
375
+ [annotation[12] * im_w, annotation[13] * im_h]],
376
+ dtype=np.float32)
377
+ return lm_align