Spaces:
Sleeping
Sleeping
editing a bunch of file paths.
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- MakeItTalk/animated.py +19 -19
- MakeItTalk/examples/dump/random_val_au.pickle +2 -2
- MakeItTalk/examples/dump/random_val_fl.pickle +2 -2
- MakeItTalk/examples/in_audio.wav +0 -0
- MakeItTalk/examples/in_audio_av.mp4 +0 -0
- MakeItTalk/examples/in_image.jpg +0 -0
- MakeItTalk/examples/in_image_pred_fls_in_audio_audio_embed.mp4 +0 -0
- MakeItTalk/main_end2end.py +18 -18
- MakeItTalk/main_end2end_cartoon.py +26 -26
- MakeItTalk/main_gen_new_puppet.py +1 -1
- MakeItTalk/marlene_test.ipynb +18 -18
- MakeItTalk/quick_demo.ipynb +18 -18
- MakeItTalk/quick_demo_tdlr.ipynb +19 -19
- MakeItTalk/src/__pycache__/__init__.cpython-37.pyc +0 -0
- MakeItTalk/src/approaches/__pycache__/__init__.cpython-37.pyc +0 -0
- MakeItTalk/src/approaches/__pycache__/train_audio2landmark.cpython-37.pyc +0 -0
- MakeItTalk/src/approaches/__pycache__/train_image_translation.cpython-37.pyc +0 -0
- MakeItTalk/src/approaches/train_audio2landmark.py +4 -4
- MakeItTalk/src/approaches/train_content.py +2 -2
- MakeItTalk/src/approaches/train_noautovc.py +1 -1
- MakeItTalk/src/approaches/train_speaker_aware.py +1 -1
- MakeItTalk/src/autovc/AutoVC_mel_Convertor_retrain_version.py +1 -1
- MakeItTalk/src/autovc/__pycache__/AutoVC_mel_Convertor_retrain_version.cpython-37.pyc +0 -0
- MakeItTalk/src/autovc/__pycache__/__init__.cpython-37.pyc +0 -0
- MakeItTalk/src/autovc/__pycache__/utils.cpython-37.pyc +0 -0
- MakeItTalk/src/autovc/retrain_version/__pycache__/__init__.cpython-37.pyc +0 -0
- MakeItTalk/src/autovc/retrain_version/__pycache__/model_vc_37_1.cpython-37.pyc +0 -0
- MakeItTalk/src/autovc/retrain_version/vocoder_spec/__pycache__/__init__.cpython-37.pyc +0 -0
- MakeItTalk/src/autovc/retrain_version/vocoder_spec/__pycache__/extract_f0_func.cpython-37.pyc +0 -0
- MakeItTalk/src/autovc/retrain_version/vocoder_spec/__pycache__/utils.cpython-37.pyc +0 -0
- MakeItTalk/src/dataset/__pycache__/__init__.cpython-37.pyc +0 -0
- MakeItTalk/src/dataset/audio2landmark/__pycache__/__init__.cpython-37.pyc +0 -0
- MakeItTalk/src/dataset/audio2landmark/__pycache__/audio2landmark_dataset.cpython-37.pyc +0 -0
- MakeItTalk/src/dataset/audio2landmark/audio2landmark_dataset.py +2 -2
- MakeItTalk/src/dataset/image_translation/__pycache__/__init__.cpython-37.pyc +0 -0
- MakeItTalk/src/dataset/image_translation/__pycache__/data_preparation.cpython-37.pyc +0 -0
- MakeItTalk/src/dataset/image_translation/__pycache__/image_translation_dataset.cpython-37.pyc +0 -0
- MakeItTalk/src/dataset/utils/__pycache__/Av2Flau_Convertor.cpython-37.pyc +0 -0
- MakeItTalk/src/dataset/utils/__pycache__/__init__.cpython-37.pyc +0 -0
- MakeItTalk/src/dataset/utils/__pycache__/icp.cpython-37.pyc +0 -0
- MakeItTalk/src/models/__pycache__/__init__.cpython-37.pyc +0 -0
- MakeItTalk/src/models/__pycache__/model_audio2landmark.cpython-37.pyc +0 -0
- MakeItTalk/src/models/__pycache__/model_image_translation.cpython-37.pyc +0 -0
- MakeItTalk/test.ipynb +19 -19
- MakeItTalk/thirdparty/AdaptiveWingLoss/.gitignore +0 -8
- MakeItTalk/thirdparty/AdaptiveWingLoss/__pycache__/__init__.cpython-37.pyc +0 -0
- MakeItTalk/thirdparty/AdaptiveWingLoss/__pycache__/__init__.cpython-39.pyc +0 -0
- MakeItTalk/thirdparty/AdaptiveWingLoss/ckpt/.gitkeep +0 -0
- MakeItTalk/thirdparty/AdaptiveWingLoss/core/__pycache__/__init__.cpython-37.pyc +0 -0
- MakeItTalk/thirdparty/AdaptiveWingLoss/core/__pycache__/__init__.cpython-39.pyc +0 -0
MakeItTalk/animated.py
CHANGED
@@ -15,7 +15,7 @@ import ipywidgets as widgets
|
|
15 |
import glob
|
16 |
import matplotlib.pyplot as plt
|
17 |
print("Choose the image name to animate: (saved in folder 'MakeItTalk/examples/')")
|
18 |
-
img_list = glob.glob1('examples', '*.jpg')
|
19 |
img_list.sort()
|
20 |
img_list = [item.split('.')[0] for item in img_list]
|
21 |
default_head_name = widgets.Dropdown(options=img_list, value='marlene_v2')
|
@@ -97,7 +97,7 @@ parser.add_argument('--amp_pos', type=float, default=AMP_HEAD_POSE_MOTION)
|
|
97 |
parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) # ['iWeklsXc0H8']) #['45hn7-LXDX8']) #['E_kmpT-EfOg']) #'iWeklsXc0H8', '29k8RtSUjE0', '45hn7-LXDX8',
|
98 |
parser.add_argument('--add_audio_in', default=False, action='store_true')
|
99 |
parser.add_argument('--comb_fan_awing', default=False, action='store_true')
|
100 |
-
parser.add_argument('--output_folder', type=str, default='examples')
|
101 |
parser.add_argument('--test_end2end', default=True, action='store_true')
|
102 |
parser.add_argument('--dump_dir', type=str, default='', help='')
|
103 |
parser.add_argument('--pos_dim', default=7, type=int)
|
@@ -148,7 +148,7 @@ print("Loaded Image...", file=sys.stderr)
|
|
148 |
# %%
|
149 |
au_data = []
|
150 |
au_emb = []
|
151 |
-
ains = glob.glob1('examples', '*.wav')
|
152 |
ains = [item for item in ains if item != 'tmp.wav']
|
153 |
ains.sort()
|
154 |
for ain in ains:
|
@@ -161,9 +161,9 @@ for ain in ains:
|
|
161 |
au_emb.append(me.reshape(-1))
|
162 |
|
163 |
print('Processing audio file', ain)
|
164 |
-
c = AutoVC_mel_Convertor('examples')
|
165 |
|
166 |
-
au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=os.path.join('examples', ain),
|
167 |
autovc_model_path=opt_parser.load_AUTOVC_name)
|
168 |
au_data += au_data_i
|
169 |
if(os.path.isfile('MakeItTalk/examples/tmp.wav')):
|
@@ -185,20 +185,20 @@ for au, info in au_data:
|
|
185 |
rot_quat.append(np.zeros(shape=(au_length, 4)))
|
186 |
anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))
|
187 |
|
188 |
-
if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl.pickle'))):
|
189 |
-
os.remove(os.path.join('examples', 'dump', 'random_val_fl.pickle'))
|
190 |
-
if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))):
|
191 |
-
os.remove(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))
|
192 |
-
if(os.path.exists(os.path.join('examples', 'dump', 'random_val_au.pickle'))):
|
193 |
-
os.remove(os.path.join('examples', 'dump', 'random_val_au.pickle'))
|
194 |
-
if (os.path.exists(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))):
|
195 |
-
os.remove(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))
|
196 |
|
197 |
-
with open(os.path.join('examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:
|
198 |
pickle.dump(fl_data, fp)
|
199 |
-
with open(os.path.join('examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:
|
200 |
pickle.dump(au_data, fp)
|
201 |
-
with open(os.path.join('examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:
|
202 |
gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}
|
203 |
pickle.dump(gaze, fp)
|
204 |
|
@@ -214,11 +214,11 @@ print("Audio->Landmark...", file=sys.stderr)
|
|
214 |
|
215 |
|
216 |
# %%
|
217 |
-
fls = glob.glob1('examples', 'pred_fls_*.txt')
|
218 |
fls.sort()
|
219 |
|
220 |
for i in range(0,len(fls)):
|
221 |
-
fl = np.loadtxt(os.path.join('examples', fls[i])).reshape((-1, 68,3))
|
222 |
print(fls[i])
|
223 |
fl[:, :, 0:2] = -fl[:, :, 0:2]
|
224 |
fl[:, :, 0:2] = fl[:, :, 0:2] / scale - shift
|
@@ -237,7 +237,7 @@ for i in range(0,len(fls)):
|
|
237 |
with torch.no_grad():
|
238 |
model.single_test(jpg=img, fls=fl, filename=fls[i], prefix=opt_parser.jpg.split('.')[0])
|
239 |
print('finish image2image gen')
|
240 |
-
os.remove(os.path.join('examples', fls[i]))
|
241 |
|
242 |
print("{} / {}: Landmark->Face...".format(i+1, len(fls)), file=sys.stderr)
|
243 |
print("Done!", file=sys.stderr)
|
|
|
15 |
import glob
|
16 |
import matplotlib.pyplot as plt
|
17 |
print("Choose the image name to animate: (saved in folder 'MakeItTalk/examples/')")
|
18 |
+
img_list = glob.glob1('MakeItTalk/examples', '*.jpg')
|
19 |
img_list.sort()
|
20 |
img_list = [item.split('.')[0] for item in img_list]
|
21 |
default_head_name = widgets.Dropdown(options=img_list, value='marlene_v2')
|
|
|
97 |
parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) # ['iWeklsXc0H8']) #['45hn7-LXDX8']) #['E_kmpT-EfOg']) #'iWeklsXc0H8', '29k8RtSUjE0', '45hn7-LXDX8',
|
98 |
parser.add_argument('--add_audio_in', default=False, action='store_true')
|
99 |
parser.add_argument('--comb_fan_awing', default=False, action='store_true')
|
100 |
+
parser.add_argument('--output_folder', type=str, default='MakeItTalk/examples')
|
101 |
parser.add_argument('--test_end2end', default=True, action='store_true')
|
102 |
parser.add_argument('--dump_dir', type=str, default='', help='')
|
103 |
parser.add_argument('--pos_dim', default=7, type=int)
|
|
|
148 |
# %%
|
149 |
au_data = []
|
150 |
au_emb = []
|
151 |
+
ains = glob.glob1('MakeItTalk/examples', '*.wav')
|
152 |
ains = [item for item in ains if item != 'tmp.wav']
|
153 |
ains.sort()
|
154 |
for ain in ains:
|
|
|
161 |
au_emb.append(me.reshape(-1))
|
162 |
|
163 |
print('Processing audio file', ain)
|
164 |
+
c = AutoVC_mel_Convertor('MakeItTalk/examples')
|
165 |
|
166 |
+
au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=os.path.join('MakeItTalk/examples', ain),
|
167 |
autovc_model_path=opt_parser.load_AUTOVC_name)
|
168 |
au_data += au_data_i
|
169 |
if(os.path.isfile('MakeItTalk/examples/tmp.wav')):
|
|
|
185 |
rot_quat.append(np.zeros(shape=(au_length, 4)))
|
186 |
anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))
|
187 |
|
188 |
+
if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'))):
|
189 |
+
os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'))
|
190 |
+
if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl_interp.pickle'))):
|
191 |
+
os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl_interp.pickle'))
|
192 |
+
if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'))):
|
193 |
+
os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'))
|
194 |
+
if (os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'))):
|
195 |
+
os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'))
|
196 |
|
197 |
+
with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:
|
198 |
pickle.dump(fl_data, fp)
|
199 |
+
with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:
|
200 |
pickle.dump(au_data, fp)
|
201 |
+
with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:
|
202 |
gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}
|
203 |
pickle.dump(gaze, fp)
|
204 |
|
|
|
214 |
|
215 |
|
216 |
# %%
|
217 |
+
fls = glob.glob1('MakeItTalk/examples', 'pred_fls_*.txt')
|
218 |
fls.sort()
|
219 |
|
220 |
for i in range(0,len(fls)):
|
221 |
+
fl = np.loadtxt(os.path.join('MakeItTalk/examples', fls[i])).reshape((-1, 68,3))
|
222 |
print(fls[i])
|
223 |
fl[:, :, 0:2] = -fl[:, :, 0:2]
|
224 |
fl[:, :, 0:2] = fl[:, :, 0:2] / scale - shift
|
|
|
237 |
with torch.no_grad():
|
238 |
model.single_test(jpg=img, fls=fl, filename=fls[i], prefix=opt_parser.jpg.split('.')[0])
|
239 |
print('finish image2image gen')
|
240 |
+
os.remove(os.path.join('MakeItTalk/examples', fls[i]))
|
241 |
|
242 |
print("{} / {}: Landmark->Face...".format(i+1, len(fls)), file=sys.stderr)
|
243 |
print("Done!", file=sys.stderr)
|
MakeItTalk/examples/dump/random_val_au.pickle
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:c067bc38f69311b87902146eb9c42f4dd06d123980b22b9476848e5451cacecf
|
3 |
+
size 98874
|
MakeItTalk/examples/dump/random_val_fl.pickle
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:c5f8aad8e9a4165ee354c96000afac730a1c01cdcbe5ca93b005e203cc3ce85c
|
3 |
+
size 499075
|
MakeItTalk/examples/in_audio.wav
ADDED
Binary file (156 kB). View file
|
|
MakeItTalk/examples/in_audio_av.mp4
ADDED
Binary file (242 kB). View file
|
|
MakeItTalk/examples/in_image.jpg
CHANGED
![]() |
![]() |
MakeItTalk/examples/in_image_pred_fls_in_audio_audio_embed.mp4
ADDED
Binary file (132 kB). View file
|
|
MakeItTalk/main_end2end.py
CHANGED
@@ -45,7 +45,7 @@ parser.add_argument('--amp_pos', type=float, default=.5)
|
|
45 |
parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) # ['iWeklsXc0H8']) #['45hn7-LXDX8']) #['E_kmpT-EfOg']) #'iWeklsXc0H8', '29k8RtSUjE0', '45hn7-LXDX8',
|
46 |
parser.add_argument('--add_audio_in', default=False, action='store_true')
|
47 |
parser.add_argument('--comb_fan_awing', default=False, action='store_true')
|
48 |
-
parser.add_argument('--output_folder', type=str, default='examples')
|
49 |
|
50 |
parser.add_argument('--test_end2end', default=True, action='store_true')
|
51 |
parser.add_argument('--dump_dir', type=str, default='', help='')
|
@@ -95,7 +95,7 @@ shape_3d, scale, shift = util.norm_input_face(shape_3d)
|
|
95 |
# audio real data
|
96 |
au_data = []
|
97 |
au_emb = []
|
98 |
-
ains = glob.glob1('examples', '*.wav')
|
99 |
ains = [item for item in ains if item != 'tmp.wav']
|
100 |
ains.sort()
|
101 |
for ain in ains:
|
@@ -108,9 +108,9 @@ for ain in ains:
|
|
108 |
au_emb.append(me.reshape(-1))
|
109 |
|
110 |
print('Processing audio file', ain)
|
111 |
-
c = AutoVC_mel_Convertor('examples')
|
112 |
|
113 |
-
au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=os.path.join('examples', ain),
|
114 |
autovc_model_path=opt_parser.load_AUTOVC_name)
|
115 |
au_data += au_data_i
|
116 |
if(os.path.isfile('MakeItTalk/examples/tmp.wav')):
|
@@ -127,20 +127,20 @@ for au, info in au_data:
|
|
127 |
rot_quat.append(np.zeros(shape=(au_length, 4)))
|
128 |
anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))
|
129 |
|
130 |
-
if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl.pickle'))):
|
131 |
-
os.remove(os.path.join('examples', 'dump', 'random_val_fl.pickle'))
|
132 |
-
if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))):
|
133 |
-
os.remove(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))
|
134 |
-
if(os.path.exists(os.path.join('examples', 'dump', 'random_val_au.pickle'))):
|
135 |
-
os.remove(os.path.join('examples', 'dump', 'random_val_au.pickle'))
|
136 |
-
if (os.path.exists(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))):
|
137 |
-
os.remove(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))
|
138 |
|
139 |
-
with open(os.path.join('examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:
|
140 |
pickle.dump(fl_data, fp)
|
141 |
-
with open(os.path.join('examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:
|
142 |
pickle.dump(au_data, fp)
|
143 |
-
with open(os.path.join('examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:
|
144 |
gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}
|
145 |
pickle.dump(gaze, fp)
|
146 |
|
@@ -154,11 +154,11 @@ else:
|
|
154 |
|
155 |
|
156 |
''' STEP 5: de-normalize the output to the original image scale '''
|
157 |
-
fls = glob.glob1('examples', 'pred_fls_*.txt') #it looks like fls is the name of our desired output video but as a group of numpy arrays in a txt file
|
158 |
fls.sort()
|
159 |
|
160 |
for i in range(0,len(fls)):
|
161 |
-
fl = np.loadtxt(os.path.join('examples', fls[i])).reshape((-1, 68,3)) #this is our desired image loaded into numpy ndarray. Data read from the text file.
|
162 |
fl[:, :, 0:2] = -fl[:, :, 0:2] #i think this is adjusting the color
|
163 |
fl[:, :, 0:2] = fl[:, :, 0:2] / scale - shift #an ndarray image array is (H, W, D) i.e. (height, width, depth), so we are adjusting depth here
|
164 |
|
@@ -176,4 +176,4 @@ for i in range(0,len(fls)):
|
|
176 |
with torch.no_grad():
|
177 |
model.single_test(jpg=img, fls=fl, filename=fls[i], prefix=opt_parser.jpg.split('.')[0]) #fls is the video we want
|
178 |
print('finish image2image gen')
|
179 |
-
os.remove(os.path.join('examples', fls[i]))
|
|
|
45 |
parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) # ['iWeklsXc0H8']) #['45hn7-LXDX8']) #['E_kmpT-EfOg']) #'iWeklsXc0H8', '29k8RtSUjE0', '45hn7-LXDX8',
|
46 |
parser.add_argument('--add_audio_in', default=False, action='store_true')
|
47 |
parser.add_argument('--comb_fan_awing', default=False, action='store_true')
|
48 |
+
parser.add_argument('--output_folder', type=str, default='MakeItTalk/examples')
|
49 |
|
50 |
parser.add_argument('--test_end2end', default=True, action='store_true')
|
51 |
parser.add_argument('--dump_dir', type=str, default='', help='')
|
|
|
95 |
# audio real data
|
96 |
au_data = []
|
97 |
au_emb = []
|
98 |
+
ains = glob.glob1('MakeItTalk/examples', '*.wav')
|
99 |
ains = [item for item in ains if item != 'tmp.wav']
|
100 |
ains.sort()
|
101 |
for ain in ains:
|
|
|
108 |
au_emb.append(me.reshape(-1))
|
109 |
|
110 |
print('Processing audio file', ain)
|
111 |
+
c = AutoVC_mel_Convertor('MakeItTalk/examples')
|
112 |
|
113 |
+
au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=os.path.join('MakeItTalk/examples', ain),
|
114 |
autovc_model_path=opt_parser.load_AUTOVC_name)
|
115 |
au_data += au_data_i
|
116 |
if(os.path.isfile('MakeItTalk/examples/tmp.wav')):
|
|
|
127 |
rot_quat.append(np.zeros(shape=(au_length, 4)))
|
128 |
anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))
|
129 |
|
130 |
+
if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'))):
|
131 |
+
os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'))
|
132 |
+
if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl_interp.pickle'))):
|
133 |
+
os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl_interp.pickle'))
|
134 |
+
if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'))):
|
135 |
+
os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'))
|
136 |
+
if (os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'))):
|
137 |
+
os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'))
|
138 |
|
139 |
+
with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:
|
140 |
pickle.dump(fl_data, fp)
|
141 |
+
with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:
|
142 |
pickle.dump(au_data, fp)
|
143 |
+
with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:
|
144 |
gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}
|
145 |
pickle.dump(gaze, fp)
|
146 |
|
|
|
154 |
|
155 |
|
156 |
''' STEP 5: de-normalize the output to the original image scale '''
|
157 |
+
fls = glob.glob1('MakeItTalk/examples', 'pred_fls_*.txt') #it looks like fls is the name of our desired output video but as a group of numpy arrays in a txt file
|
158 |
fls.sort()
|
159 |
|
160 |
for i in range(0,len(fls)):
|
161 |
+
fl = np.loadtxt(os.path.join('MakeItTalk/examples', fls[i])).reshape((-1, 68,3)) #this is our desired image loaded into numpy ndarray. Data read from the text file.
|
162 |
fl[:, :, 0:2] = -fl[:, :, 0:2] #i think this is adjusting the color
|
163 |
fl[:, :, 0:2] = fl[:, :, 0:2] / scale - shift #an ndarray image array is (H, W, D) i.e. (height, width, depth), so we are adjusting depth here
|
164 |
|
|
|
176 |
with torch.no_grad():
|
177 |
model.single_test(jpg=img, fls=fl, filename=fls[i], prefix=opt_parser.jpg.split('.')[0]) #fls is the video we want
|
178 |
print('finish image2image gen')
|
179 |
+
os.remove(os.path.join('MakeItTalk/examples', fls[i]))
|
MakeItTalk/main_end2end_cartoon.py
CHANGED
@@ -43,7 +43,7 @@ parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) #
|
|
43 |
|
44 |
parser.add_argument('--add_audio_in', default=False, action='store_true')
|
45 |
parser.add_argument('--comb_fan_awing', default=False, action='store_true')
|
46 |
-
parser.add_argument('--output_folder', type=str, default='examples_cartoon')
|
47 |
|
48 |
#### NEW POSE MODEL
|
49 |
parser.add_argument('--test_end2end', default=True, action='store_true')
|
@@ -68,12 +68,12 @@ opt_parser = parser.parse_args()
|
|
68 |
|
69 |
DEMO_CH = opt_parser.jpg.split('.')[0]
|
70 |
|
71 |
-
shape_3d = np.loadtxt('examples_cartoon/{}_face_close_mouth.txt'.format(DEMO_CH))
|
72 |
|
73 |
''' STEP 3: Generate audio data as input to audio branch '''
|
74 |
au_data = []
|
75 |
au_emb = []
|
76 |
-
ains = glob.glob1('examples', '*.wav')
|
77 |
ains = [item for item in ains if item is not 'tmp.wav']
|
78 |
ains.sort()
|
79 |
for ain in ains:
|
@@ -86,11 +86,11 @@ for ain in ains:
|
|
86 |
au_emb.append(me.reshape(-1))
|
87 |
|
88 |
print('Processing audio file', ain)
|
89 |
-
c = AutoVC_mel_Convertor('examples')
|
90 |
-
au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=os.path.join('examples', ain),
|
91 |
autovc_model_path=opt_parser.load_AUTOVC_name)
|
92 |
au_data += au_data_i
|
93 |
-
# os.remove(os.path.join('examples', 'tmp.wav'))
|
94 |
if(os.path.isfile('MakeItTalk/examples/tmp.wav')):
|
95 |
os.remove('MakeItTalk/examples/tmp.wav')
|
96 |
|
@@ -104,20 +104,20 @@ for au, info in au_data:
|
|
104 |
rot_quat.append(np.zeros(shape=(au_length, 4)))
|
105 |
anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))
|
106 |
|
107 |
-
if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl.pickle'))):
|
108 |
-
os.remove(os.path.join('examples', 'dump', 'random_val_fl.pickle'))
|
109 |
-
if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))):
|
110 |
-
os.remove(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))
|
111 |
-
if(os.path.exists(os.path.join('examples', 'dump', 'random_val_au.pickle'))):
|
112 |
-
os.remove(os.path.join('examples', 'dump', 'random_val_au.pickle'))
|
113 |
-
if (os.path.exists(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))):
|
114 |
-
os.remove(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))
|
115 |
|
116 |
-
with open(os.path.join('examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:
|
117 |
pickle.dump(fl_data, fp)
|
118 |
-
with open(os.path.join('examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:
|
119 |
pickle.dump(au_data, fp)
|
120 |
-
with open(os.path.join('examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:
|
121 |
gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}
|
122 |
pickle.dump(gaze, fp)
|
123 |
|
@@ -132,15 +132,15 @@ else:
|
|
132 |
print('finish gen fls')
|
133 |
|
134 |
''' STEP 5: de-normalize the output to the original image scale '''
|
135 |
-
fls_names = glob.glob1('examples_cartoon', 'pred_fls_*.txt')
|
136 |
fls_names.sort()
|
137 |
|
138 |
for i in range(0,len(fls_names)):
|
139 |
-
ains = glob.glob1('examples', '*.wav')
|
140 |
ains.sort()
|
141 |
ain = ains[i]
|
142 |
-
fl = np.loadtxt(os.path.join('examples_cartoon', fls_names[i])).reshape((-1, 68,3))
|
143 |
-
output_dir = os.path.join('examples_cartoon', fls_names[i][:-4])
|
144 |
try:
|
145 |
os.makedirs(output_dir)
|
146 |
except:
|
@@ -148,7 +148,7 @@ for i in range(0,len(fls_names)):
|
|
148 |
|
149 |
from util.utils import get_puppet_info
|
150 |
|
151 |
-
bound, scale, shift = get_puppet_info(DEMO_CH, ROOT_DIR='examples_cartoon')
|
152 |
|
153 |
fls = fl.reshape((-1, 68, 3))
|
154 |
|
@@ -182,16 +182,16 @@ for i in range(0,len(fls_names)):
|
|
182 |
np.savetxt(os.path.join(output_dir, 'warped_points.txt'), fls, fmt='%.2f')
|
183 |
|
184 |
# static_points.txt
|
185 |
-
static_frame = np.loadtxt(os.path.join('examples_cartoon', '{}_face_open_mouth.txt'.format(DEMO_CH)))
|
186 |
static_frame = static_frame[r, 0:2]
|
187 |
static_frame = np.concatenate((static_frame, bound.reshape(-1, 2)), axis=0)
|
188 |
np.savetxt(os.path.join(output_dir, 'reference_points.txt'), static_frame, fmt='%.2f')
|
189 |
|
190 |
# triangle_vtx_index.txt
|
191 |
-
shutil.copy(os.path.join('examples_cartoon', DEMO_CH + '_delauney_tri.txt'),
|
192 |
os.path.join(output_dir, 'triangulation.txt'))
|
193 |
|
194 |
-
os.remove(os.path.join('examples_cartoon', fls_names[i]))
|
195 |
|
196 |
# ==============================================
|
197 |
# Step 4 : Vector art morphing
|
@@ -227,6 +227,6 @@ for i in range(0,len(fls_names)):
|
|
227 |
os.path.join(cur_dir, '..', '..', opt_parser.jpg_bg),
|
228 |
'-novsync -dump'))
|
229 |
os.system('ffmpeg -y -r 62.5 -f image2 -i "%06d.tga" -i {} -pix_fmt yuv420p -vf "pad=ceil(iw/2)*2:ceil(ih/2)*2" -shortest -strict -2 {}'.format(
|
230 |
-
os.path.join(cur_dir, '..', '..', '..', 'examples', ain),
|
231 |
os.path.join(cur_dir, '..', 'out.mp4')
|
232 |
))
|
|
|
43 |
|
44 |
parser.add_argument('--add_audio_in', default=False, action='store_true')
|
45 |
parser.add_argument('--comb_fan_awing', default=False, action='store_true')
|
46 |
+
parser.add_argument('--output_folder', type=str, default='MakeItTalk/examples_cartoon')
|
47 |
|
48 |
#### NEW POSE MODEL
|
49 |
parser.add_argument('--test_end2end', default=True, action='store_true')
|
|
|
68 |
|
69 |
DEMO_CH = opt_parser.jpg.split('.')[0]
|
70 |
|
71 |
+
shape_3d = np.loadtxt('MakeItTalk/examples_cartoon/{}_face_close_mouth.txt'.format(DEMO_CH))
|
72 |
|
73 |
''' STEP 3: Generate audio data as input to audio branch '''
|
74 |
au_data = []
|
75 |
au_emb = []
|
76 |
+
ains = glob.glob1('MakeItTalk/examples', '*.wav')
|
77 |
ains = [item for item in ains if item is not 'tmp.wav']
|
78 |
ains.sort()
|
79 |
for ain in ains:
|
|
|
86 |
au_emb.append(me.reshape(-1))
|
87 |
|
88 |
print('Processing audio file', ain)
|
89 |
+
c = AutoVC_mel_Convertor('MakeItTalk/examples')
|
90 |
+
au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=os.path.join('MakeItTalk/examples', ain),
|
91 |
autovc_model_path=opt_parser.load_AUTOVC_name)
|
92 |
au_data += au_data_i
|
93 |
+
# os.remove(os.path.join('MakeItTalk/examples', 'tmp.wav'))
|
94 |
if(os.path.isfile('MakeItTalk/examples/tmp.wav')):
|
95 |
os.remove('MakeItTalk/examples/tmp.wav')
|
96 |
|
|
|
104 |
rot_quat.append(np.zeros(shape=(au_length, 4)))
|
105 |
anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))
|
106 |
|
107 |
+
if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'))):
|
108 |
+
os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'))
|
109 |
+
if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl_interp.pickle'))):
|
110 |
+
os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl_interp.pickle'))
|
111 |
+
if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'))):
|
112 |
+
os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'))
|
113 |
+
if (os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'))):
|
114 |
+
os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'))
|
115 |
|
116 |
+
with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:
|
117 |
pickle.dump(fl_data, fp)
|
118 |
+
with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:
|
119 |
pickle.dump(au_data, fp)
|
120 |
+
with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:
|
121 |
gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}
|
122 |
pickle.dump(gaze, fp)
|
123 |
|
|
|
132 |
print('finish gen fls')
|
133 |
|
134 |
''' STEP 5: de-normalize the output to the original image scale '''
|
135 |
+
fls_names = glob.glob1('MakeItTalk/examples_cartoon', 'pred_fls_*.txt')
|
136 |
fls_names.sort()
|
137 |
|
138 |
for i in range(0,len(fls_names)):
|
139 |
+
ains = glob.glob1('MakeItTalk/examples', '*.wav')
|
140 |
ains.sort()
|
141 |
ain = ains[i]
|
142 |
+
fl = np.loadtxt(os.path.join('MakeItTalk/examples_cartoon', fls_names[i])).reshape((-1, 68,3))
|
143 |
+
output_dir = os.path.join('MakeItTalk/examples_cartoon', fls_names[i][:-4])
|
144 |
try:
|
145 |
os.makedirs(output_dir)
|
146 |
except:
|
|
|
148 |
|
149 |
from util.utils import get_puppet_info
|
150 |
|
151 |
+
bound, scale, shift = get_puppet_info(DEMO_CH, ROOT_DIR='MakeItTalk/examples_cartoon')
|
152 |
|
153 |
fls = fl.reshape((-1, 68, 3))
|
154 |
|
|
|
182 |
np.savetxt(os.path.join(output_dir, 'warped_points.txt'), fls, fmt='%.2f')
|
183 |
|
184 |
# static_points.txt
|
185 |
+
static_frame = np.loadtxt(os.path.join('MakeItTalk/examples_cartoon', '{}_face_open_mouth.txt'.format(DEMO_CH)))
|
186 |
static_frame = static_frame[r, 0:2]
|
187 |
static_frame = np.concatenate((static_frame, bound.reshape(-1, 2)), axis=0)
|
188 |
np.savetxt(os.path.join(output_dir, 'reference_points.txt'), static_frame, fmt='%.2f')
|
189 |
|
190 |
# triangle_vtx_index.txt
|
191 |
+
shutil.copy(os.path.join('MakeItTalk/examples_cartoon', DEMO_CH + '_delauney_tri.txt'),
|
192 |
os.path.join(output_dir, 'triangulation.txt'))
|
193 |
|
194 |
+
os.remove(os.path.join('MakeItTalk/examples_cartoon', fls_names[i]))
|
195 |
|
196 |
# ==============================================
|
197 |
# Step 4 : Vector art morphing
|
|
|
227 |
os.path.join(cur_dir, '..', '..', opt_parser.jpg_bg),
|
228 |
'-novsync -dump'))
|
229 |
os.system('ffmpeg -y -r 62.5 -f image2 -i "%06d.tga" -i {} -pix_fmt yuv420p -vf "pad=ceil(iw/2)*2:ceil(ih/2)*2" -shortest -strict -2 {}'.format(
|
230 |
+
os.path.join(cur_dir, '..', '..', '..', 'MakeItTalk/examples', ain),
|
231 |
os.path.join(cur_dir, '..', 'out.mp4')
|
232 |
))
|
MakeItTalk/main_gen_new_puppet.py
CHANGED
@@ -5,7 +5,7 @@ from facewarp.gen_puppet_utils import *
|
|
5 |
FOA face landmark detection
|
6 |
================================================ '''
|
7 |
|
8 |
-
data_dir = out_dir = 'examples_cartoon'
|
9 |
test_data = sys.argv[1] # for example 'roy_example.png'
|
10 |
CH = test_data[:-4]
|
11 |
use_gt_bb = False
|
|
|
5 |
FOA face landmark detection
|
6 |
================================================ '''
|
7 |
|
8 |
+
data_dir = out_dir = 'MakeItTalk/examples_cartoon'
|
9 |
test_data = sys.argv[1] # for example 'roy_example.png'
|
10 |
CH = test_data[:-4]
|
11 |
use_gt_bb = False
|
MakeItTalk/marlene_test.ipynb
CHANGED
@@ -61,7 +61,7 @@
|
|
61 |
"source": [
|
62 |
"\n",
|
63 |
"# print(\"Choose the image name to animate: (saved in folder 'MakeItTalk/examples/')\")\n",
|
64 |
-
"# img_list = glob.glob1('examples', '*.jpg')\n",
|
65 |
"# img_list.sort()\n",
|
66 |
"# img_list = [item.split('.')[0] for item in img_list]\n",
|
67 |
"# default_head_name = widgets.Dropdown(options=img_list, value='marlene_v2')\n",
|
@@ -136,7 +136,7 @@
|
|
136 |
"parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) # ['iWeklsXc0H8']) #['45hn7-LXDX8']) #['E_kmpT-EfOg']) #'iWeklsXc0H8', '29k8RtSUjE0', '45hn7-LXDX8',\n",
|
137 |
"parser.add_argument('--add_audio_in', default=False, action='store_true')\n",
|
138 |
"parser.add_argument('--comb_fan_awing', default=False, action='store_true')\n",
|
139 |
-
"parser.add_argument('--output_folder', type=str, default='examples')\n",
|
140 |
"parser.add_argument('--test_end2end', default=True, action='store_true')\n",
|
141 |
"parser.add_argument('--dump_dir', type=str, default='', help='')\n",
|
142 |
"parser.add_argument('--pos_dim', default=7, type=int)\n",
|
@@ -259,7 +259,7 @@
|
|
259 |
"#now we want to load the audio file \n",
|
260 |
"# au_data = []\n",
|
261 |
"# au_emb = []\n",
|
262 |
-
"# ains = glob.glob1('examples', '*.wav')\n",
|
263 |
"# ains = [item for item in ains if item != 'tmp.wav']\n",
|
264 |
"# ains.sort()\n",
|
265 |
"\n",
|
@@ -274,7 +274,7 @@
|
|
274 |
"me, ae = get_spk_emb(f'MakeItTalk/examples/{input_audio}')\n",
|
275 |
"au_emb.append(me.reshape(-1))\n",
|
276 |
"\n",
|
277 |
-
"c = AutoVC_mel_Convertor('examples')\n",
|
278 |
"\n",
|
279 |
"au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=input_audio, autovc_model_path=opt_parser.load_AUTOVC_name)\n",
|
280 |
"\n",
|
@@ -302,20 +302,20 @@
|
|
302 |
" rot_quat.append(np.zeros(shape=(au_length, 4)))\n",
|
303 |
" anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))\n",
|
304 |
"\n",
|
305 |
-
"if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl.pickle'))):\n",
|
306 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_fl.pickle'))\n",
|
307 |
-
"if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))):\n",
|
308 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))\n",
|
309 |
-
"if(os.path.exists(os.path.join('examples', 'dump', 'random_val_au.pickle'))):\n",
|
310 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_au.pickle'))\n",
|
311 |
-
"if (os.path.exists(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))):\n",
|
312 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))\n",
|
313 |
"\n",
|
314 |
-
"with open(os.path.join('examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:\n",
|
315 |
" pickle.dump(fl_data, fp)\n",
|
316 |
-
"with open(os.path.join('examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:\n",
|
317 |
" pickle.dump(au_data, fp)\n",
|
318 |
-
"with open(os.path.join('examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:\n",
|
319 |
" gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}\n",
|
320 |
" pickle.dump(gaze, fp)"
|
321 |
]
|
@@ -435,11 +435,11 @@
|
|
435 |
}
|
436 |
],
|
437 |
"source": [
|
438 |
-
"fls = glob.glob1('examples', 'pred_fls_*.txt')\n",
|
439 |
"fls.sort()\n",
|
440 |
"\n",
|
441 |
"for i in range(0,len(fls)):\n",
|
442 |
-
" fl = np.loadtxt(os.path.join('examples', fls[i])).reshape((-1, 68,3))\n",
|
443 |
" print(fls[i])\n",
|
444 |
" fl[:, :, 0:2] = -fl[:, :, 0:2]\n",
|
445 |
" fl[:, :, 0:2] = fl[:, :, 0:2] / scale - shift\n",
|
@@ -458,7 +458,7 @@
|
|
458 |
" with torch.no_grad():\n",
|
459 |
" model.single_test(jpg=img, fls=fl, filename=fls[i], prefix=opt_parser.jpg.split('.')[0])\n",
|
460 |
" print('finish image2image gen')\n",
|
461 |
-
" os.remove(os.path.join('examples', fls[i]))\n",
|
462 |
"\n",
|
463 |
" print(\"{} / {}: Landmark->Face...\".format(i+1, len(fls)), file=sys.stderr)\n",
|
464 |
"print(\"Done!\", file=sys.stderr)"
|
|
|
61 |
"source": [
|
62 |
"\n",
|
63 |
"# print(\"Choose the image name to animate: (saved in folder 'MakeItTalk/examples/')\")\n",
|
64 |
+
"# img_list = glob.glob1('MakeItTalk/examples', '*.jpg')\n",
|
65 |
"# img_list.sort()\n",
|
66 |
"# img_list = [item.split('.')[0] for item in img_list]\n",
|
67 |
"# default_head_name = widgets.Dropdown(options=img_list, value='marlene_v2')\n",
|
|
|
136 |
"parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) # ['iWeklsXc0H8']) #['45hn7-LXDX8']) #['E_kmpT-EfOg']) #'iWeklsXc0H8', '29k8RtSUjE0', '45hn7-LXDX8',\n",
|
137 |
"parser.add_argument('--add_audio_in', default=False, action='store_true')\n",
|
138 |
"parser.add_argument('--comb_fan_awing', default=False, action='store_true')\n",
|
139 |
+
"parser.add_argument('--output_folder', type=str, default='MakeItTalk/examples')\n",
|
140 |
"parser.add_argument('--test_end2end', default=True, action='store_true')\n",
|
141 |
"parser.add_argument('--dump_dir', type=str, default='', help='')\n",
|
142 |
"parser.add_argument('--pos_dim', default=7, type=int)\n",
|
|
|
259 |
"#now we want to load the audio file \n",
|
260 |
"# au_data = []\n",
|
261 |
"# au_emb = []\n",
|
262 |
+
"# ains = glob.glob1('MakeItTalk/examples', '*.wav')\n",
|
263 |
"# ains = [item for item in ains if item != 'tmp.wav']\n",
|
264 |
"# ains.sort()\n",
|
265 |
"\n",
|
|
|
274 |
"me, ae = get_spk_emb(f'MakeItTalk/examples/{input_audio}')\n",
|
275 |
"au_emb.append(me.reshape(-1))\n",
|
276 |
"\n",
|
277 |
+
"c = AutoVC_mel_Convertor('MakeItTalk/examples')\n",
|
278 |
"\n",
|
279 |
"au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=input_audio, autovc_model_path=opt_parser.load_AUTOVC_name)\n",
|
280 |
"\n",
|
|
|
302 |
" rot_quat.append(np.zeros(shape=(au_length, 4)))\n",
|
303 |
" anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))\n",
|
304 |
"\n",
|
305 |
+
"if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'))):\n",
|
306 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'))\n",
|
307 |
+
"if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl_interp.pickle'))):\n",
|
308 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl_interp.pickle'))\n",
|
309 |
+
"if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'))):\n",
|
310 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'))\n",
|
311 |
+
"if (os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'))):\n",
|
312 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'))\n",
|
313 |
"\n",
|
314 |
+
"with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:\n",
|
315 |
" pickle.dump(fl_data, fp)\n",
|
316 |
+
"with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:\n",
|
317 |
" pickle.dump(au_data, fp)\n",
|
318 |
+
"with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:\n",
|
319 |
" gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}\n",
|
320 |
" pickle.dump(gaze, fp)"
|
321 |
]
|
|
|
435 |
}
|
436 |
],
|
437 |
"source": [
|
438 |
+
"fls = glob.glob1('MakeItTalk/examples', 'pred_fls_*.txt')\n",
|
439 |
"fls.sort()\n",
|
440 |
"\n",
|
441 |
"for i in range(0,len(fls)):\n",
|
442 |
+
" fl = np.loadtxt(os.path.join('MakeItTalk/examples', fls[i])).reshape((-1, 68,3))\n",
|
443 |
" print(fls[i])\n",
|
444 |
" fl[:, :, 0:2] = -fl[:, :, 0:2]\n",
|
445 |
" fl[:, :, 0:2] = fl[:, :, 0:2] / scale - shift\n",
|
|
|
458 |
" with torch.no_grad():\n",
|
459 |
" model.single_test(jpg=img, fls=fl, filename=fls[i], prefix=opt_parser.jpg.split('.')[0])\n",
|
460 |
" print('finish image2image gen')\n",
|
461 |
+
" os.remove(os.path.join('MakeItTalk/examples', fls[i]))\n",
|
462 |
"\n",
|
463 |
" print(\"{} / {}: Landmark->Face...\".format(i+1, len(fls)), file=sys.stderr)\n",
|
464 |
"print(\"Done!\", file=sys.stderr)"
|
MakeItTalk/quick_demo.ipynb
CHANGED
@@ -414,7 +414,7 @@
|
|
414 |
"parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) # ['iWeklsXc0H8']) #['45hn7-LXDX8']) #['E_kmpT-EfOg']) #'iWeklsXc0H8', '29k8RtSUjE0', '45hn7-LXDX8',\n",
|
415 |
"parser.add_argument('--add_audio_in', default=False, action='store_true')\n",
|
416 |
"parser.add_argument('--comb_fan_awing', default=False, action='store_true')\n",
|
417 |
-
"parser.add_argument('--output_folder', type=str, default='examples')\n",
|
418 |
"\n",
|
419 |
"parser.add_argument('--test_end2end', default=True, action='store_true')\n",
|
420 |
"parser.add_argument('--dump_dir', type=str, default='', help='')\n",
|
@@ -536,7 +536,7 @@
|
|
536 |
"source": [
|
537 |
"au_data = []\n",
|
538 |
"au_emb = []\n",
|
539 |
-
"ains = glob.glob1('examples', '*.wav')\n",
|
540 |
"ains = [item for item in ains if item is not 'tmp.wav']\n",
|
541 |
"ains.sort()\n",
|
542 |
"for ain in ains:\n",
|
@@ -549,9 +549,9 @@
|
|
549 |
" au_emb.append(me.reshape(-1))\n",
|
550 |
"\n",
|
551 |
" print('Processing audio file', ain)\n",
|
552 |
-
" c = AutoVC_mel_Convertor('examples')\n",
|
553 |
"\n",
|
554 |
-
" au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=os.path.join('examples', ain),\n",
|
555 |
" autovc_model_path=opt_parser.load_AUTOVC_name)\n",
|
556 |
" au_data += au_data_i\n",
|
557 |
"if(os.path.isfile('MakeItTalk/examples/tmp.wav')):\n",
|
@@ -568,20 +568,20 @@
|
|
568 |
" rot_quat.append(np.zeros(shape=(au_length, 4)))\n",
|
569 |
" anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))\n",
|
570 |
"\n",
|
571 |
-
"if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl.pickle'))):\n",
|
572 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_fl.pickle'))\n",
|
573 |
-
"if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))):\n",
|
574 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))\n",
|
575 |
-
"if(os.path.exists(os.path.join('examples', 'dump', 'random_val_au.pickle'))):\n",
|
576 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_au.pickle'))\n",
|
577 |
-
"if (os.path.exists(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))):\n",
|
578 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))\n",
|
579 |
"\n",
|
580 |
-
"with open(os.path.join('examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:\n",
|
581 |
" pickle.dump(fl_data, fp)\n",
|
582 |
-
"with open(os.path.join('examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:\n",
|
583 |
" pickle.dump(au_data, fp)\n",
|
584 |
-
"with open(os.path.join('examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:\n",
|
585 |
" gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}\n",
|
586 |
" pickle.dump(gaze, fp)"
|
587 |
],
|
@@ -747,11 +747,11 @@
|
|
747 |
}
|
748 |
},
|
749 |
"source": [
|
750 |
-
"fls = glob.glob1('examples', 'pred_fls_*.txt')\n",
|
751 |
"fls.sort()\n",
|
752 |
"\n",
|
753 |
"for i in range(0,len(fls)):\n",
|
754 |
-
" fl = np.loadtxt(os.path.join('examples', fls[i])).reshape((-1, 68,3))\n",
|
755 |
" fl[:, :, 0:2] = -fl[:, :, 0:2]\n",
|
756 |
" fl[:, :, 0:2] = fl[:, :, 0:2] / scale - shift\n",
|
757 |
"\n",
|
@@ -769,7 +769,7 @@
|
|
769 |
" with torch.no_grad():\n",
|
770 |
" model.single_test(jpg=img, fls=fl, filename=fls[i], prefix=opt_parser.jpg.split('.')[0])\n",
|
771 |
" print('finish image2image gen')\n",
|
772 |
-
" os.remove(os.path.join('examples', fls[i]))"
|
773 |
],
|
774 |
"execution_count": null,
|
775 |
"outputs": [
|
|
|
414 |
"parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) # ['iWeklsXc0H8']) #['45hn7-LXDX8']) #['E_kmpT-EfOg']) #'iWeklsXc0H8', '29k8RtSUjE0', '45hn7-LXDX8',\n",
|
415 |
"parser.add_argument('--add_audio_in', default=False, action='store_true')\n",
|
416 |
"parser.add_argument('--comb_fan_awing', default=False, action='store_true')\n",
|
417 |
+
"parser.add_argument('--output_folder', type=str, default='MakeItTalk/examples')\n",
|
418 |
"\n",
|
419 |
"parser.add_argument('--test_end2end', default=True, action='store_true')\n",
|
420 |
"parser.add_argument('--dump_dir', type=str, default='', help='')\n",
|
|
|
536 |
"source": [
|
537 |
"au_data = []\n",
|
538 |
"au_emb = []\n",
|
539 |
+
"ains = glob.glob1('MakeItTalk/examples', '*.wav')\n",
|
540 |
"ains = [item for item in ains if item is not 'tmp.wav']\n",
|
541 |
"ains.sort()\n",
|
542 |
"for ain in ains:\n",
|
|
|
549 |
" au_emb.append(me.reshape(-1))\n",
|
550 |
"\n",
|
551 |
" print('Processing audio file', ain)\n",
|
552 |
+
" c = AutoVC_mel_Convertor('MakeItTalk/examples')\n",
|
553 |
"\n",
|
554 |
+
" au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=os.path.join('MakeItTalk/examples', ain),\n",
|
555 |
" autovc_model_path=opt_parser.load_AUTOVC_name)\n",
|
556 |
" au_data += au_data_i\n",
|
557 |
"if(os.path.isfile('MakeItTalk/examples/tmp.wav')):\n",
|
|
|
568 |
" rot_quat.append(np.zeros(shape=(au_length, 4)))\n",
|
569 |
" anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))\n",
|
570 |
"\n",
|
571 |
+
"if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'))):\n",
|
572 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'))\n",
|
573 |
+
"if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl_interp.pickle'))):\n",
|
574 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl_interp.pickle'))\n",
|
575 |
+
"if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'))):\n",
|
576 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'))\n",
|
577 |
+
"if (os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'))):\n",
|
578 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'))\n",
|
579 |
"\n",
|
580 |
+
"with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:\n",
|
581 |
" pickle.dump(fl_data, fp)\n",
|
582 |
+
"with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:\n",
|
583 |
" pickle.dump(au_data, fp)\n",
|
584 |
+
"with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:\n",
|
585 |
" gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}\n",
|
586 |
" pickle.dump(gaze, fp)"
|
587 |
],
|
|
|
747 |
}
|
748 |
},
|
749 |
"source": [
|
750 |
+
"fls = glob.glob1('MakeItTalk/examples', 'pred_fls_*.txt')\n",
|
751 |
"fls.sort()\n",
|
752 |
"\n",
|
753 |
"for i in range(0,len(fls)):\n",
|
754 |
+
" fl = np.loadtxt(os.path.join('MakeItTalk/examples', fls[i])).reshape((-1, 68,3))\n",
|
755 |
" fl[:, :, 0:2] = -fl[:, :, 0:2]\n",
|
756 |
" fl[:, :, 0:2] = fl[:, :, 0:2] / scale - shift\n",
|
757 |
"\n",
|
|
|
769 |
" with torch.no_grad():\n",
|
770 |
" model.single_test(jpg=img, fls=fl, filename=fls[i], prefix=opt_parser.jpg.split('.')[0])\n",
|
771 |
" print('finish image2image gen')\n",
|
772 |
+
" os.remove(os.path.join('MakeItTalk/examples', fls[i]))"
|
773 |
],
|
774 |
"execution_count": null,
|
775 |
"outputs": [
|
MakeItTalk/quick_demo_tdlr.ipynb
CHANGED
@@ -187,7 +187,7 @@
|
|
187 |
"import glob\n",
|
188 |
"import matplotlib.pyplot as plt\n",
|
189 |
"print(\"Choose the image name to animate: (saved in folder 'MakeItTalk/examples/')\")\n",
|
190 |
-
"img_list = glob.glob1('examples', '*.jpg')\n",
|
191 |
"img_list.sort()\n",
|
192 |
"img_list = [item.split('.')[0] for item in img_list]\n",
|
193 |
"default_head_name = widgets.Dropdown(options=img_list, value='paint_boy')\n",
|
@@ -316,7 +316,7 @@
|
|
316 |
"parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) # ['iWeklsXc0H8']) #['45hn7-LXDX8']) #['E_kmpT-EfOg']) #'iWeklsXc0H8', '29k8RtSUjE0', '45hn7-LXDX8',\n",
|
317 |
"parser.add_argument('--add_audio_in', default=False, action='store_true')\n",
|
318 |
"parser.add_argument('--comb_fan_awing', default=False, action='store_true')\n",
|
319 |
-
"parser.add_argument('--output_folder', type=str, default='examples')\n",
|
320 |
"parser.add_argument('--test_end2end', default=True, action='store_true')\n",
|
321 |
"parser.add_argument('--dump_dir', type=str, default='', help='')\n",
|
322 |
"parser.add_argument('--pos_dim', default=7, type=int)\n",
|
@@ -356,7 +356,7 @@
|
|
356 |
"\n",
|
357 |
"au_data = []\n",
|
358 |
"au_emb = []\n",
|
359 |
-
"ains = glob.glob1('examples', '*.wav')\n",
|
360 |
"ains = [item for item in ains if item is not 'tmp.wav']\n",
|
361 |
"ains.sort()\n",
|
362 |
"for ain in ains:\n",
|
@@ -369,9 +369,9 @@
|
|
369 |
" au_emb.append(me.reshape(-1))\n",
|
370 |
"\n",
|
371 |
" print('Processing audio file', ain)\n",
|
372 |
-
" c = AutoVC_mel_Convertor('examples')\n",
|
373 |
"\n",
|
374 |
-
" au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=os.path.join('examples', ain),\n",
|
375 |
" autovc_model_path=opt_parser.load_AUTOVC_name)\n",
|
376 |
" au_data += au_data_i\n",
|
377 |
"if(os.path.isfile('MakeItTalk/examples/tmp.wav')):\n",
|
@@ -390,20 +390,20 @@
|
|
390 |
" rot_quat.append(np.zeros(shape=(au_length, 4)))\n",
|
391 |
" anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))\n",
|
392 |
"\n",
|
393 |
-
"if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl.pickle'))):\n",
|
394 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_fl.pickle'))\n",
|
395 |
-
"if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))):\n",
|
396 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))\n",
|
397 |
-
"if(os.path.exists(os.path.join('examples', 'dump', 'random_val_au.pickle'))):\n",
|
398 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_au.pickle'))\n",
|
399 |
-
"if (os.path.exists(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))):\n",
|
400 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))\n",
|
401 |
"\n",
|
402 |
-
"with open(os.path.join('examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:\n",
|
403 |
" pickle.dump(fl_data, fp)\n",
|
404 |
-
"with open(os.path.join('examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:\n",
|
405 |
" pickle.dump(au_data, fp)\n",
|
406 |
-
"with open(os.path.join('examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:\n",
|
407 |
" gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}\n",
|
408 |
" pickle.dump(gaze, fp)\n",
|
409 |
"\n",
|
@@ -415,11 +415,11 @@
|
|
415 |
"\n",
|
416 |
"print(\"Audio->Landmark...\", file=sys.stderr)\n",
|
417 |
"\n",
|
418 |
-
"fls = glob.glob1('examples', 'pred_fls_*.txt')\n",
|
419 |
"fls.sort()\n",
|
420 |
"\n",
|
421 |
"for i in range(0,len(fls)):\n",
|
422 |
-
" fl = np.loadtxt(os.path.join('examples', fls[i])).reshape((-1, 68,3))\n",
|
423 |
" fl[:, :, 0:2] = -fl[:, :, 0:2]\n",
|
424 |
" fl[:, :, 0:2] = fl[:, :, 0:2] / scale - shift\n",
|
425 |
"\n",
|
@@ -437,7 +437,7 @@
|
|
437 |
" with torch.no_grad():\n",
|
438 |
" model.single_test(jpg=img, fls=fl, filename=fls[i], prefix=opt_parser.jpg.split('.')[0])\n",
|
439 |
" print('finish image2image gen')\n",
|
440 |
-
" os.remove(os.path.join('examples', fls[i]))\n",
|
441 |
"\n",
|
442 |
" print(\"{} / {}: Landmark->Face...\".format(i+1, len(fls)), file=sys.stderr)\n",
|
443 |
"print(\"Done!\", file=sys.stderr)"
|
|
|
187 |
"import glob\n",
|
188 |
"import matplotlib.pyplot as plt\n",
|
189 |
"print(\"Choose the image name to animate: (saved in folder 'MakeItTalk/examples/')\")\n",
|
190 |
+
"img_list = glob.glob1('MakeItTalk/examples', '*.jpg')\n",
|
191 |
"img_list.sort()\n",
|
192 |
"img_list = [item.split('.')[0] for item in img_list]\n",
|
193 |
"default_head_name = widgets.Dropdown(options=img_list, value='paint_boy')\n",
|
|
|
316 |
"parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) # ['iWeklsXc0H8']) #['45hn7-LXDX8']) #['E_kmpT-EfOg']) #'iWeklsXc0H8', '29k8RtSUjE0', '45hn7-LXDX8',\n",
|
317 |
"parser.add_argument('--add_audio_in', default=False, action='store_true')\n",
|
318 |
"parser.add_argument('--comb_fan_awing', default=False, action='store_true')\n",
|
319 |
+
"parser.add_argument('--output_folder', type=str, default='MakeItTalk/examples')\n",
|
320 |
"parser.add_argument('--test_end2end', default=True, action='store_true')\n",
|
321 |
"parser.add_argument('--dump_dir', type=str, default='', help='')\n",
|
322 |
"parser.add_argument('--pos_dim', default=7, type=int)\n",
|
|
|
356 |
"\n",
|
357 |
"au_data = []\n",
|
358 |
"au_emb = []\n",
|
359 |
+
"ains = glob.glob1('MakeItTalk/examples', '*.wav')\n",
|
360 |
"ains = [item for item in ains if item is not 'tmp.wav']\n",
|
361 |
"ains.sort()\n",
|
362 |
"for ain in ains:\n",
|
|
|
369 |
" au_emb.append(me.reshape(-1))\n",
|
370 |
"\n",
|
371 |
" print('Processing audio file', ain)\n",
|
372 |
+
" c = AutoVC_mel_Convertor('MakeItTalk/examples')\n",
|
373 |
"\n",
|
374 |
+
" au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=os.path.join('MakeItTalk/examples', ain),\n",
|
375 |
" autovc_model_path=opt_parser.load_AUTOVC_name)\n",
|
376 |
" au_data += au_data_i\n",
|
377 |
"if(os.path.isfile('MakeItTalk/examples/tmp.wav')):\n",
|
|
|
390 |
" rot_quat.append(np.zeros(shape=(au_length, 4)))\n",
|
391 |
" anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))\n",
|
392 |
"\n",
|
393 |
+
"if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'))):\n",
|
394 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'))\n",
|
395 |
+
"if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl_interp.pickle'))):\n",
|
396 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl_interp.pickle'))\n",
|
397 |
+
"if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'))):\n",
|
398 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'))\n",
|
399 |
+
"if (os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'))):\n",
|
400 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'))\n",
|
401 |
"\n",
|
402 |
+
"with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:\n",
|
403 |
" pickle.dump(fl_data, fp)\n",
|
404 |
+
"with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:\n",
|
405 |
" pickle.dump(au_data, fp)\n",
|
406 |
+
"with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:\n",
|
407 |
" gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}\n",
|
408 |
" pickle.dump(gaze, fp)\n",
|
409 |
"\n",
|
|
|
415 |
"\n",
|
416 |
"print(\"Audio->Landmark...\", file=sys.stderr)\n",
|
417 |
"\n",
|
418 |
+
"fls = glob.glob1('MakeItTalk/examples', 'pred_fls_*.txt')\n",
|
419 |
"fls.sort()\n",
|
420 |
"\n",
|
421 |
"for i in range(0,len(fls)):\n",
|
422 |
+
" fl = np.loadtxt(os.path.join('MakeItTalk/examples', fls[i])).reshape((-1, 68,3))\n",
|
423 |
" fl[:, :, 0:2] = -fl[:, :, 0:2]\n",
|
424 |
" fl[:, :, 0:2] = fl[:, :, 0:2] / scale - shift\n",
|
425 |
"\n",
|
|
|
437 |
" with torch.no_grad():\n",
|
438 |
" model.single_test(jpg=img, fls=fl, filename=fls[i], prefix=opt_parser.jpg.split('.')[0])\n",
|
439 |
" print('finish image2image gen')\n",
|
440 |
+
" os.remove(os.path.join('MakeItTalk/examples', fls[i]))\n",
|
441 |
"\n",
|
442 |
" print(\"{} / {}: Landmark->Face...\".format(i+1, len(fls)), file=sys.stderr)\n",
|
443 |
"print(\"Done!\", file=sys.stderr)"
|
MakeItTalk/src/__pycache__/__init__.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/__pycache__/__init__.cpython-37.pyc and b/MakeItTalk/src/__pycache__/__init__.cpython-37.pyc differ
|
|
MakeItTalk/src/approaches/__pycache__/__init__.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/approaches/__pycache__/__init__.cpython-37.pyc and b/MakeItTalk/src/approaches/__pycache__/__init__.cpython-37.pyc differ
|
|
MakeItTalk/src/approaches/__pycache__/train_audio2landmark.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/approaches/__pycache__/train_audio2landmark.cpython-37.pyc and b/MakeItTalk/src/approaches/__pycache__/train_audio2landmark.cpython-37.pyc differ
|
|
MakeItTalk/src/approaches/__pycache__/train_image_translation.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/approaches/__pycache__/train_image_translation.cpython-37.pyc and b/MakeItTalk/src/approaches/__pycache__/train_image_translation.cpython-37.pyc differ
|
|
MakeItTalk/src/approaches/train_audio2landmark.py
CHANGED
@@ -30,7 +30,7 @@ class Audio2landmark_model():
|
|
30 |
|
31 |
# Step 1 : load opt_parser
|
32 |
self.opt_parser = opt_parser
|
33 |
-
self.std_face_id = np.loadtxt('src/dataset/utils/STD_FACE_LANDMARKS.txt')
|
34 |
if(jpg_shape is not None):
|
35 |
self.std_face_id = jpg_shape
|
36 |
self.std_face_id = self.std_face_id.reshape(1, 204)
|
@@ -75,10 +75,10 @@ class Audio2landmark_model():
|
|
75 |
self.C.to(device)
|
76 |
|
77 |
self.t_shape_idx = (27, 28, 29, 30, 33, 36, 39, 42, 45)
|
78 |
-
self.anchor_t_shape = np.loadtxt('src/dataset/utils/STD_FACE_LANDMARKS.txt')
|
79 |
self.anchor_t_shape = self.anchor_t_shape[self.t_shape_idx, :]
|
80 |
|
81 |
-
with open(os.path.join('examples', 'dump', 'emb.pickle'), 'rb') as fp:
|
82 |
self.test_embs = pickle.load(fp)
|
83 |
|
84 |
print('====================================')
|
@@ -247,7 +247,7 @@ class Audio2landmark_model():
|
|
247 |
if(vis_fls):
|
248 |
from util.vis import Vis
|
249 |
Vis(fls=fake_fls_np, filename=video_name.split('\\')[-1].split('/')[-1], fps=62.5,
|
250 |
-
audio_filenam=os.path.join('examples', video_name.split('\\')[-1].split('/')[-1]+'.wav'))
|
251 |
|
252 |
|
253 |
def __close_face_lip__(self, fl):
|
|
|
30 |
|
31 |
# Step 1 : load opt_parser
|
32 |
self.opt_parser = opt_parser
|
33 |
+
self.std_face_id = np.loadtxt('MakeItTalk/src/dataset/utils/STD_FACE_LANDMARKS.txt')
|
34 |
if(jpg_shape is not None):
|
35 |
self.std_face_id = jpg_shape
|
36 |
self.std_face_id = self.std_face_id.reshape(1, 204)
|
|
|
75 |
self.C.to(device)
|
76 |
|
77 |
self.t_shape_idx = (27, 28, 29, 30, 33, 36, 39, 42, 45)
|
78 |
+
self.anchor_t_shape = np.loadtxt('MakeItTalk/src/dataset/utils/STD_FACE_LANDMARKS.txt')
|
79 |
self.anchor_t_shape = self.anchor_t_shape[self.t_shape_idx, :]
|
80 |
|
81 |
+
with open(os.path.join('MakeItTalk/examples', 'dump', 'emb.pickle'), 'rb') as fp:
|
82 |
self.test_embs = pickle.load(fp)
|
83 |
|
84 |
print('====================================')
|
|
|
247 |
if(vis_fls):
|
248 |
from util.vis import Vis
|
249 |
Vis(fls=fake_fls_np, filename=video_name.split('\\')[-1].split('/')[-1], fps=62.5,
|
250 |
+
audio_filenam=os.path.join('MakeItTalk/examples', video_name.split('\\')[-1].split('/')[-1]+'.wav'))
|
251 |
|
252 |
|
253 |
def __close_face_lip__(self, fl):
|
MakeItTalk/src/approaches/train_content.py
CHANGED
@@ -32,7 +32,7 @@ class Audio2landmark_model():
|
|
32 |
|
33 |
# Step 1 : load opt_parser
|
34 |
self.opt_parser = opt_parser
|
35 |
-
self.std_face_id = np.loadtxt('src/dataset/utils/STD_FACE_LANDMARKS.txt')
|
36 |
if(jpg_shape is not None):
|
37 |
self.std_face_id = jpg_shape
|
38 |
self.std_face_id = self.std_face_id.reshape(1, 204)
|
@@ -70,7 +70,7 @@ class Audio2landmark_model():
|
|
70 |
self.C.to(device)
|
71 |
|
72 |
self.t_shape_idx = (27, 28, 29, 30, 33, 36, 39, 42, 45)
|
73 |
-
self.anchor_t_shape = np.loadtxt('src/dataset/utils/STD_FACE_LANDMARKS.txt')
|
74 |
self.anchor_t_shape = self.anchor_t_shape[self.t_shape_idx, :]
|
75 |
|
76 |
self.opt_C = optim.Adam(self.C.parameters(), lr=opt_parser.lr, weight_decay=opt_parser.reg_lr)
|
|
|
32 |
|
33 |
# Step 1 : load opt_parser
|
34 |
self.opt_parser = opt_parser
|
35 |
+
self.std_face_id = np.loadtxt('MakeItTalk/src/dataset/utils/STD_FACE_LANDMARKS.txt')
|
36 |
if(jpg_shape is not None):
|
37 |
self.std_face_id = jpg_shape
|
38 |
self.std_face_id = self.std_face_id.reshape(1, 204)
|
|
|
70 |
self.C.to(device)
|
71 |
|
72 |
self.t_shape_idx = (27, 28, 29, 30, 33, 36, 39, 42, 45)
|
73 |
+
self.anchor_t_shape = np.loadtxt('MakeItTalk/src/dataset/utils/STD_FACE_LANDMARKS.txt')
|
74 |
self.anchor_t_shape = self.anchor_t_shape[self.t_shape_idx, :]
|
75 |
|
76 |
self.opt_C = optim.Adam(self.C.parameters(), lr=opt_parser.lr, weight_decay=opt_parser.reg_lr)
|
MakeItTalk/src/approaches/train_noautovc.py
CHANGED
@@ -381,7 +381,7 @@ class Speaker_aware_branch():
|
|
381 |
|
382 |
fake_fls_np = np.concatenate(fls_pred_pos_list)
|
383 |
filename = 'pred_fls_{}_{}.txt'.format(video_name.split('/')[-1], key)
|
384 |
-
np.savetxt(os.path.join('examples', filename), fake_fls_np, fmt='%.6f')
|
385 |
|
386 |
|
387 |
def __close_face_lip__(self, fl):
|
|
|
381 |
|
382 |
fake_fls_np = np.concatenate(fls_pred_pos_list)
|
383 |
filename = 'pred_fls_{}_{}.txt'.format(video_name.split('/')[-1], key)
|
384 |
+
np.savetxt(os.path.join('MakeItTalk/examples', filename), fake_fls_np, fmt='%.6f')
|
385 |
|
386 |
|
387 |
def __close_face_lip__(self, fl):
|
MakeItTalk/src/approaches/train_speaker_aware.py
CHANGED
@@ -604,7 +604,7 @@ class Speaker_aware_branch():
|
|
604 |
|
605 |
fake_fls_np = np.concatenate(fls_pred_pos_list)
|
606 |
filename = 'pred_fls_{}_{}.txt'.format(video_name.split('/')[-1], key)
|
607 |
-
np.savetxt(os.path.join('examples', filename), fake_fls_np, fmt='%.6f')
|
608 |
|
609 |
|
610 |
def __close_face_lip__(self, fl):
|
|
|
604 |
|
605 |
fake_fls_np = np.concatenate(fls_pred_pos_list)
|
606 |
filename = 'pred_fls_{}_{}.txt'.format(video_name.split('/')[-1], key)
|
607 |
+
np.savetxt(os.path.join('MakeItTalk/examples', filename), fake_fls_np, fmt='%.6f')
|
608 |
|
609 |
|
610 |
def __close_face_lip__(self, fl):
|
MakeItTalk/src/autovc/AutoVC_mel_Convertor_retrain_version.py
CHANGED
@@ -211,7 +211,7 @@ class AutoVC_mel_Convertor():
|
|
211 |
g_checkpoint = torch.load(autovc_model_path, map_location=device)
|
212 |
G.load_state_dict(g_checkpoint['model'])
|
213 |
|
214 |
-
emb = np.loadtxt('src/autovc/retrain_version/obama_emb.txt')
|
215 |
emb_trg = torch.from_numpy(emb[np.newaxis, :].astype('float32')).to(device)
|
216 |
|
217 |
aus = []
|
|
|
211 |
g_checkpoint = torch.load(autovc_model_path, map_location=device)
|
212 |
G.load_state_dict(g_checkpoint['model'])
|
213 |
|
214 |
+
emb = np.loadtxt('MakeItTalk/src/autovc/retrain_version/obama_emb.txt')
|
215 |
emb_trg = torch.from_numpy(emb[np.newaxis, :].astype('float32')).to(device)
|
216 |
|
217 |
aus = []
|
MakeItTalk/src/autovc/__pycache__/AutoVC_mel_Convertor_retrain_version.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/autovc/__pycache__/AutoVC_mel_Convertor_retrain_version.cpython-37.pyc and b/MakeItTalk/src/autovc/__pycache__/AutoVC_mel_Convertor_retrain_version.cpython-37.pyc differ
|
|
MakeItTalk/src/autovc/__pycache__/__init__.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/autovc/__pycache__/__init__.cpython-37.pyc and b/MakeItTalk/src/autovc/__pycache__/__init__.cpython-37.pyc differ
|
|
MakeItTalk/src/autovc/__pycache__/utils.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/autovc/__pycache__/utils.cpython-37.pyc and b/MakeItTalk/src/autovc/__pycache__/utils.cpython-37.pyc differ
|
|
MakeItTalk/src/autovc/retrain_version/__pycache__/__init__.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/autovc/retrain_version/__pycache__/__init__.cpython-37.pyc and b/MakeItTalk/src/autovc/retrain_version/__pycache__/__init__.cpython-37.pyc differ
|
|
MakeItTalk/src/autovc/retrain_version/__pycache__/model_vc_37_1.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/autovc/retrain_version/__pycache__/model_vc_37_1.cpython-37.pyc and b/MakeItTalk/src/autovc/retrain_version/__pycache__/model_vc_37_1.cpython-37.pyc differ
|
|
MakeItTalk/src/autovc/retrain_version/vocoder_spec/__pycache__/__init__.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/autovc/retrain_version/vocoder_spec/__pycache__/__init__.cpython-37.pyc and b/MakeItTalk/src/autovc/retrain_version/vocoder_spec/__pycache__/__init__.cpython-37.pyc differ
|
|
MakeItTalk/src/autovc/retrain_version/vocoder_spec/__pycache__/extract_f0_func.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/autovc/retrain_version/vocoder_spec/__pycache__/extract_f0_func.cpython-37.pyc and b/MakeItTalk/src/autovc/retrain_version/vocoder_spec/__pycache__/extract_f0_func.cpython-37.pyc differ
|
|
MakeItTalk/src/autovc/retrain_version/vocoder_spec/__pycache__/utils.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/autovc/retrain_version/vocoder_spec/__pycache__/utils.cpython-37.pyc and b/MakeItTalk/src/autovc/retrain_version/vocoder_spec/__pycache__/utils.cpython-37.pyc differ
|
|
MakeItTalk/src/dataset/__pycache__/__init__.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/dataset/__pycache__/__init__.cpython-37.pyc and b/MakeItTalk/src/dataset/__pycache__/__init__.cpython-37.pyc differ
|
|
MakeItTalk/src/dataset/audio2landmark/__pycache__/__init__.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/dataset/audio2landmark/__pycache__/__init__.cpython-37.pyc and b/MakeItTalk/src/dataset/audio2landmark/__pycache__/__init__.cpython-37.pyc differ
|
|
MakeItTalk/src/dataset/audio2landmark/__pycache__/audio2landmark_dataset.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/dataset/audio2landmark/__pycache__/audio2landmark_dataset.cpython-37.pyc and b/MakeItTalk/src/dataset/audio2landmark/__pycache__/audio2landmark_dataset.cpython-37.pyc differ
|
|
MakeItTalk/src/dataset/audio2landmark/audio2landmark_dataset.py
CHANGED
@@ -42,7 +42,7 @@ class Audio2landmark_Dataset(data.Dataset):
|
|
42 |
self.fl_data = [self.fl_data[i] for i in valid_idx]
|
43 |
self.au_data = [self.au_data[i] for i in valid_idx]
|
44 |
|
45 |
-
au_mean_std = np.loadtxt('src/dataset/utils/MEAN_STD_AUTOVC_RETRAIN_MEL_AU.txt')
|
46 |
au_mean, au_std = au_mean_std[0:au_mean_std.shape[0]//2], au_mean_std[au_mean_std.shape[0]//2:]
|
47 |
|
48 |
self.au_data = [((au - au_mean) / au_std, info) for au, info in self.au_data]
|
@@ -229,7 +229,7 @@ class Speaker_aware_branch_Dataset(data.Dataset):
|
|
229 |
# print('SAVE!')
|
230 |
|
231 |
|
232 |
-
au_mean_std = np.loadtxt('src/dataset/utils/MEAN_STD_AUTOVC_RETRAIN_MEL_AU.txt') # np.mean(self.au_data[0][0]), np.std(self.au_data[0][0])
|
233 |
au_mean, au_std = au_mean_std[0:au_mean_std.shape[0]//2], au_mean_std[au_mean_std.shape[0]//2:]
|
234 |
|
235 |
self.au_data = [((au - au_mean) / au_std, info) for au, info in self.au_data]
|
|
|
42 |
self.fl_data = [self.fl_data[i] for i in valid_idx]
|
43 |
self.au_data = [self.au_data[i] for i in valid_idx]
|
44 |
|
45 |
+
au_mean_std = np.loadtxt('MakeItTalk/src/dataset/utils/MEAN_STD_AUTOVC_RETRAIN_MEL_AU.txt')
|
46 |
au_mean, au_std = au_mean_std[0:au_mean_std.shape[0]//2], au_mean_std[au_mean_std.shape[0]//2:]
|
47 |
|
48 |
self.au_data = [((au - au_mean) / au_std, info) for au, info in self.au_data]
|
|
|
229 |
# print('SAVE!')
|
230 |
|
231 |
|
232 |
+
au_mean_std = np.loadtxt('MakeItTalk/src/dataset/utils/MEAN_STD_AUTOVC_RETRAIN_MEL_AU.txt') # np.mean(self.au_data[0][0]), np.std(self.au_data[0][0])
|
233 |
au_mean, au_std = au_mean_std[0:au_mean_std.shape[0]//2], au_mean_std[au_mean_std.shape[0]//2:]
|
234 |
|
235 |
self.au_data = [((au - au_mean) / au_std, info) for au, info in self.au_data]
|
MakeItTalk/src/dataset/image_translation/__pycache__/__init__.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/dataset/image_translation/__pycache__/__init__.cpython-37.pyc and b/MakeItTalk/src/dataset/image_translation/__pycache__/__init__.cpython-37.pyc differ
|
|
MakeItTalk/src/dataset/image_translation/__pycache__/data_preparation.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/dataset/image_translation/__pycache__/data_preparation.cpython-37.pyc and b/MakeItTalk/src/dataset/image_translation/__pycache__/data_preparation.cpython-37.pyc differ
|
|
MakeItTalk/src/dataset/image_translation/__pycache__/image_translation_dataset.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/dataset/image_translation/__pycache__/image_translation_dataset.cpython-37.pyc and b/MakeItTalk/src/dataset/image_translation/__pycache__/image_translation_dataset.cpython-37.pyc differ
|
|
MakeItTalk/src/dataset/utils/__pycache__/Av2Flau_Convertor.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/dataset/utils/__pycache__/Av2Flau_Convertor.cpython-37.pyc and b/MakeItTalk/src/dataset/utils/__pycache__/Av2Flau_Convertor.cpython-37.pyc differ
|
|
MakeItTalk/src/dataset/utils/__pycache__/__init__.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/dataset/utils/__pycache__/__init__.cpython-37.pyc and b/MakeItTalk/src/dataset/utils/__pycache__/__init__.cpython-37.pyc differ
|
|
MakeItTalk/src/dataset/utils/__pycache__/icp.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/dataset/utils/__pycache__/icp.cpython-37.pyc and b/MakeItTalk/src/dataset/utils/__pycache__/icp.cpython-37.pyc differ
|
|
MakeItTalk/src/models/__pycache__/__init__.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/models/__pycache__/__init__.cpython-37.pyc and b/MakeItTalk/src/models/__pycache__/__init__.cpython-37.pyc differ
|
|
MakeItTalk/src/models/__pycache__/model_audio2landmark.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/models/__pycache__/model_audio2landmark.cpython-37.pyc and b/MakeItTalk/src/models/__pycache__/model_audio2landmark.cpython-37.pyc differ
|
|
MakeItTalk/src/models/__pycache__/model_image_translation.cpython-37.pyc
CHANGED
Binary files a/MakeItTalk/src/models/__pycache__/model_image_translation.cpython-37.pyc and b/MakeItTalk/src/models/__pycache__/model_image_translation.cpython-37.pyc differ
|
|
MakeItTalk/test.ipynb
CHANGED
@@ -60,7 +60,7 @@
|
|
60 |
"from IPython.core.debugger import Pdb;\n",
|
61 |
"\n",
|
62 |
"print(\"Choose the image name to animate: (saved in folder 'MakeItTalk/examples/')\")\n",
|
63 |
-
"img_list = glob.glob1('examples', '*.jpg')\n",
|
64 |
"img_list.sort()\n",
|
65 |
"img_list = [item.split('.')[0] for item in img_list]\n",
|
66 |
"default_head_name = widgets.Dropdown(options=img_list, value='marlene_v2')\n",
|
@@ -157,7 +157,7 @@
|
|
157 |
"parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) # ['iWeklsXc0H8']) #['45hn7-LXDX8']) #['E_kmpT-EfOg']) #'iWeklsXc0H8', '29k8RtSUjE0', '45hn7-LXDX8',\n",
|
158 |
"parser.add_argument('--add_audio_in', default=False, action='store_true') #we're using an audio file \n",
|
159 |
"parser.add_argument('--comb_fan_awing', default=False, action='store_true') #not sure \n",
|
160 |
-
"parser.add_argument('--output_folder', type=str, default='examples') #which folder to store in \n",
|
161 |
"parser.add_argument('--test_end2end', default=True, action='store_true') \n",
|
162 |
"parser.add_argument('--dump_dir', type=str, default='', help='')\n",
|
163 |
"parser.add_argument('--pos_dim', default=7, type=int)\n",
|
@@ -277,7 +277,7 @@
|
|
277 |
"source": [
|
278 |
"au_data = []\n",
|
279 |
"au_emb = []\n",
|
280 |
-
"ains = glob.glob1('examples', '*.wav')\n",
|
281 |
"ains = [item for item in ains if item != 'tmp.wav']\n",
|
282 |
"ains.sort()\n",
|
283 |
"for ain in ains:\n",
|
@@ -290,9 +290,9 @@
|
|
290 |
" au_emb.append(me.reshape(-1))\n",
|
291 |
"\n",
|
292 |
" print('Processing audio file', ain)\n",
|
293 |
-
" c = AutoVC_mel_Convertor('examples')\n",
|
294 |
"\n",
|
295 |
-
" au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=os.path.join('examples', ain),\n",
|
296 |
" autovc_model_path=opt_parser.load_AUTOVC_name)\n",
|
297 |
" au_data += au_data_i\n",
|
298 |
"print(f'this is {au_data}')\n",
|
@@ -320,20 +320,20 @@
|
|
320 |
" rot_quat.append(np.zeros(shape=(au_length, 4)))\n",
|
321 |
" anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))\n",
|
322 |
"\n",
|
323 |
-
"if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl.pickle'))):\n",
|
324 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_fl.pickle'))\n",
|
325 |
-
"if(os.path.exists(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))):\n",
|
326 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_fl_interp.pickle'))\n",
|
327 |
-
"if(os.path.exists(os.path.join('examples', 'dump', 'random_val_au.pickle'))):\n",
|
328 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_au.pickle'))\n",
|
329 |
-
"if (os.path.exists(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))):\n",
|
330 |
-
" os.remove(os.path.join('examples', 'dump', 'random_val_gaze.pickle'))\n",
|
331 |
"\n",
|
332 |
-
"with open(os.path.join('examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:\n",
|
333 |
" pickle.dump(fl_data, fp)\n",
|
334 |
-
"with open(os.path.join('examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:\n",
|
335 |
" pickle.dump(au_data, fp)\n",
|
336 |
-
"with open(os.path.join('examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:\n",
|
337 |
" gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}\n",
|
338 |
" pickle.dump(gaze, fp)"
|
339 |
]
|
@@ -442,11 +442,11 @@
|
|
442 |
"outputs": [],
|
443 |
"source": [
|
444 |
"#reshaping the data that we got \n",
|
445 |
-
"fls = glob.glob1('examples', 'pred_fls_*.txt')\n",
|
446 |
"fls.sort()\n",
|
447 |
"\n",
|
448 |
"for i in range(0,len(fls)):\n",
|
449 |
-
" fl = np.loadtxt(os.path.join('examples', fls[i])).reshape((-1, 68,3))\n",
|
450 |
" print(fls[i])\n",
|
451 |
" fl[:, :, 0:2] = -fl[:, :, 0:2]\n",
|
452 |
" fl[:, :, 0:2] = fl[:, :, 0:2] / scale - shift\n",
|
@@ -475,7 +475,7 @@
|
|
475 |
" with torch.no_grad():\n",
|
476 |
" model.single_test(jpg=img, fls=fl, filename=fls[i], prefix=opt_parser.jpg.split('.')[0])\n",
|
477 |
" print('finish image2image gen')\n",
|
478 |
-
" os.remove(os.path.join('examples', fls[i]))\n",
|
479 |
"\n",
|
480 |
" print(\"{} / {}: Landmark->Face...\".format(i+1, len(fls)), file=sys.stderr)\n",
|
481 |
"print(\"Done!\", file=sys.stderr)"
|
|
|
60 |
"from IPython.core.debugger import Pdb;\n",
|
61 |
"\n",
|
62 |
"print(\"Choose the image name to animate: (saved in folder 'MakeItTalk/examples/')\")\n",
|
63 |
+
"img_list = glob.glob1('MakeItTalk/examples', '*.jpg')\n",
|
64 |
"img_list.sort()\n",
|
65 |
"img_list = [item.split('.')[0] for item in img_list]\n",
|
66 |
"default_head_name = widgets.Dropdown(options=img_list, value='marlene_v2')\n",
|
|
|
157 |
"parser.add_argument('--reuse_train_emb_list', type=str, nargs='+', default=[]) # ['iWeklsXc0H8']) #['45hn7-LXDX8']) #['E_kmpT-EfOg']) #'iWeklsXc0H8', '29k8RtSUjE0', '45hn7-LXDX8',\n",
|
158 |
"parser.add_argument('--add_audio_in', default=False, action='store_true') #we're using an audio file \n",
|
159 |
"parser.add_argument('--comb_fan_awing', default=False, action='store_true') #not sure \n",
|
160 |
+
"parser.add_argument('--output_folder', type=str, default='MakeItTalk/examples') #which folder to store in \n",
|
161 |
"parser.add_argument('--test_end2end', default=True, action='store_true') \n",
|
162 |
"parser.add_argument('--dump_dir', type=str, default='', help='')\n",
|
163 |
"parser.add_argument('--pos_dim', default=7, type=int)\n",
|
|
|
277 |
"source": [
|
278 |
"au_data = []\n",
|
279 |
"au_emb = []\n",
|
280 |
+
"ains = glob.glob1('MakeItTalk/examples', '*.wav')\n",
|
281 |
"ains = [item for item in ains if item != 'tmp.wav']\n",
|
282 |
"ains.sort()\n",
|
283 |
"for ain in ains:\n",
|
|
|
290 |
" au_emb.append(me.reshape(-1))\n",
|
291 |
"\n",
|
292 |
" print('Processing audio file', ain)\n",
|
293 |
+
" c = AutoVC_mel_Convertor('MakeItTalk/examples')\n",
|
294 |
"\n",
|
295 |
+
" au_data_i = c.convert_single_wav_to_autovc_input(audio_filename=os.path.join('MakeItTalk/examples', ain),\n",
|
296 |
" autovc_model_path=opt_parser.load_AUTOVC_name)\n",
|
297 |
" au_data += au_data_i\n",
|
298 |
"print(f'this is {au_data}')\n",
|
|
|
320 |
" rot_quat.append(np.zeros(shape=(au_length, 4)))\n",
|
321 |
" anchor_t_shape.append(np.zeros(shape=(au_length, 68 * 3)))\n",
|
322 |
"\n",
|
323 |
+
"if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'))):\n",
|
324 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'))\n",
|
325 |
+
"if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl_interp.pickle'))):\n",
|
326 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl_interp.pickle'))\n",
|
327 |
+
"if(os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'))):\n",
|
328 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'))\n",
|
329 |
+
"if (os.path.exists(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'))):\n",
|
330 |
+
" os.remove(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'))\n",
|
331 |
"\n",
|
332 |
+
"with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_fl.pickle'), 'wb') as fp:\n",
|
333 |
" pickle.dump(fl_data, fp)\n",
|
334 |
+
"with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_au.pickle'), 'wb') as fp:\n",
|
335 |
" pickle.dump(au_data, fp)\n",
|
336 |
+
"with open(os.path.join('MakeItTalk/examples', 'dump', 'random_val_gaze.pickle'), 'wb') as fp:\n",
|
337 |
" gaze = {'rot_trans':rot_tran, 'rot_quat':rot_quat, 'anchor_t_shape':anchor_t_shape}\n",
|
338 |
" pickle.dump(gaze, fp)"
|
339 |
]
|
|
|
442 |
"outputs": [],
|
443 |
"source": [
|
444 |
"#reshaping the data that we got \n",
|
445 |
+
"fls = glob.glob1('MakeItTalk/examples', 'pred_fls_*.txt')\n",
|
446 |
"fls.sort()\n",
|
447 |
"\n",
|
448 |
"for i in range(0,len(fls)):\n",
|
449 |
+
" fl = np.loadtxt(os.path.join('MakeItTalk/examples', fls[i])).reshape((-1, 68,3))\n",
|
450 |
" print(fls[i])\n",
|
451 |
" fl[:, :, 0:2] = -fl[:, :, 0:2]\n",
|
452 |
" fl[:, :, 0:2] = fl[:, :, 0:2] / scale - shift\n",
|
|
|
475 |
" with torch.no_grad():\n",
|
476 |
" model.single_test(jpg=img, fls=fl, filename=fls[i], prefix=opt_parser.jpg.split('.')[0])\n",
|
477 |
" print('finish image2image gen')\n",
|
478 |
+
" os.remove(os.path.join('MakeItTalk/examples', fls[i]))\n",
|
479 |
"\n",
|
480 |
" print(\"{} / {}: Landmark->Face...\".format(i+1, len(fls)), file=sys.stderr)\n",
|
481 |
"print(\"Done!\", file=sys.stderr)"
|
MakeItTalk/thirdparty/AdaptiveWingLoss/.gitignore
DELETED
@@ -1,8 +0,0 @@
|
|
1 |
-
# Python generated files
|
2 |
-
*.pyc
|
3 |
-
|
4 |
-
# Project related files
|
5 |
-
ckpt/*.pth
|
6 |
-
dataset/*
|
7 |
-
!dataset/!.py
|
8 |
-
experiments/*
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
MakeItTalk/thirdparty/AdaptiveWingLoss/__pycache__/__init__.cpython-37.pyc
ADDED
Binary file (180 Bytes). View file
|
|
MakeItTalk/thirdparty/AdaptiveWingLoss/__pycache__/__init__.cpython-39.pyc
ADDED
Binary file (179 Bytes). View file
|
|
MakeItTalk/thirdparty/AdaptiveWingLoss/ckpt/.gitkeep
DELETED
File without changes
|
MakeItTalk/thirdparty/AdaptiveWingLoss/core/__pycache__/__init__.cpython-37.pyc
ADDED
Binary file (185 Bytes). View file
|
|
MakeItTalk/thirdparty/AdaptiveWingLoss/core/__pycache__/__init__.cpython-39.pyc
ADDED
Binary file (184 Bytes). View file
|
|