|
""" |
|
@paper: GAN Prior Embedded Network for Blind Face Restoration in the Wild (CVPR2021) |
|
@author: yangxy (yangtao9009@gmail.com) |
|
""" |
|
import os |
|
import cv2 |
|
import glob |
|
import time |
|
import argparse |
|
import numpy as np |
|
from PIL import Image |
|
from skimage import transform as tf |
|
|
|
import GPEN.__init_paths as init_paths |
|
from GPEN.retinaface.retinaface_detection import RetinaFaceDetection |
|
from GPEN.face_model.face_gan import FaceGAN |
|
from GPEN.sr_model.real_esrnet import RealESRNet |
|
from GPEN.align_faces import warp_and_crop_face, get_reference_facial_points |
|
|
|
def check_ckpts(model, sr_model): |
|
|
|
try: |
|
ckpts_folder = os.path.join(os.path.dirname(__file__), "weights") |
|
if not os.path.exists(ckpts_folder): |
|
print("Downloading checkpoints...") |
|
from gdown import download_folder |
|
file_id = "1epln5c8HW1QXfVz6444Fe0hG-vRNavi6" |
|
download_folder(id=file_id, output=ckpts_folder, quiet=False, use_cookies=False) |
|
else: |
|
print("Checkpoints already downloaded, skipping...") |
|
except Exception as e: |
|
print(e) |
|
raise Exception("Error while downloading checkpoints") |
|
|
|
|
|
class FaceEnhancement(object): |
|
def __init__(self, base_dir=os.path.dirname(__file__), size=512, model=None, use_sr=True, sr_model=None, channel_multiplier=2, narrow=1, use_facegan=True): |
|
check_ckpts(model, sr_model) |
|
|
|
self.facedetector = RetinaFaceDetection(base_dir) |
|
self.facegan = FaceGAN(base_dir, size, model, channel_multiplier, narrow) |
|
self.srmodel = RealESRNet(base_dir, sr_model) |
|
self.use_sr = use_sr |
|
self.size = size |
|
self.threshold = 0.9 |
|
self.use_facegan = use_facegan |
|
|
|
|
|
self.mask = np.zeros((512, 512), np.float32) |
|
cv2.rectangle(self.mask, (26, 26), (486, 486), (1, 1, 1), -1, cv2.LINE_AA) |
|
self.mask = cv2.GaussianBlur(self.mask, (101, 101), 11) |
|
self.mask = cv2.GaussianBlur(self.mask, (101, 101), 11) |
|
|
|
self.kernel = np.array(([0.0625, 0.125, 0.0625], [0.125, 0.25, 0.125], [0.0625, 0.125, 0.0625]), dtype="float32") |
|
|
|
|
|
default_square = True |
|
inner_padding_factor = 0.25 |
|
outer_padding = (0, 0) |
|
self.reference_5pts = get_reference_facial_points((self.size, self.size), inner_padding_factor, outer_padding, default_square) |
|
|
|
def process(self, img): |
|
if self.use_sr: |
|
img_sr = self.srmodel.process(img) |
|
if img_sr is not None: |
|
img = cv2.resize(img, img_sr.shape[:2][::-1]) |
|
|
|
facebs, landms = self.facedetector.detect(img) |
|
|
|
orig_faces, enhanced_faces = [], [] |
|
height, width = img.shape[:2] |
|
full_mask = np.zeros((height, width), dtype=np.float32) |
|
full_img = np.zeros(img.shape, dtype=np.uint8) |
|
|
|
for i, (faceb, facial5points) in enumerate(zip(facebs, landms)): |
|
if faceb[4] < self.threshold: |
|
continue |
|
fh, fw = (faceb[3] - faceb[1]), (faceb[2] - faceb[0]) |
|
|
|
facial5points = np.reshape(facial5points, (2, 5)) |
|
|
|
of, tfm_inv = warp_and_crop_face(img, facial5points, reference_pts=self.reference_5pts, crop_size=(self.size, self.size)) |
|
|
|
|
|
ef = self.facegan.process(of) if self.use_facegan else of |
|
|
|
orig_faces.append(of) |
|
enhanced_faces.append(ef) |
|
|
|
tmp_mask = self.mask |
|
tmp_mask = cv2.resize(tmp_mask, ef.shape[:2]) |
|
tmp_mask = cv2.warpAffine(tmp_mask, tfm_inv, (width, height), flags=3) |
|
|
|
if min(fh, fw) < 100: |
|
ef = cv2.filter2D(ef, -1, self.kernel) |
|
|
|
tmp_img = cv2.warpAffine(ef, tfm_inv, (width, height), flags=3) |
|
|
|
mask = tmp_mask - full_mask |
|
full_mask[np.where(mask > 0)] = tmp_mask[np.where(mask > 0)] |
|
full_img[np.where(mask > 0)] = tmp_img[np.where(mask > 0)] |
|
|
|
full_mask = full_mask[:, :, np.newaxis] |
|
if self.use_sr and img_sr is not None: |
|
img = cv2.convertScaleAbs(img_sr * (1 - full_mask) + full_img * full_mask) |
|
else: |
|
img = cv2.convertScaleAbs(img * (1 - full_mask) + full_img * full_mask) |
|
|
|
return img, orig_faces, enhanced_faces |
|
|
|
|
|
if __name__ == "__main__": |
|
parser = argparse.ArgumentParser() |
|
parser.add_argument("--model", type=str, default="GPEN-BFR-512", help="GPEN model") |
|
parser.add_argument("--size", type=int, default=512, help="resolution of GPEN") |
|
parser.add_argument("--channel_multiplier", type=int, default=2, help="channel multiplier of GPEN") |
|
parser.add_argument("--narrow", type=float, default=1, help="channel narrow scale") |
|
parser.add_argument("--use_sr", action="store_true", help="use sr or not") |
|
parser.add_argument("--sr_model", type=str, default="realesrnet_x2", help="SR model") |
|
parser.add_argument("--sr_scale", type=int, default=2, help="SR scale") |
|
parser.add_argument("--indir", type=str, default="examples/imgs", help="input folder") |
|
parser.add_argument("--outdir", type=str, default="results/outs-BFR", help="output folder") |
|
args = parser.parse_args() |
|
|
|
|
|
|
|
|
|
os.makedirs(args.outdir, exist_ok=True) |
|
|
|
faceenhancer = FaceEnhancement( |
|
size=args.size, |
|
model=args.model, |
|
use_sr=args.use_sr, |
|
sr_model=args.sr_model, |
|
channel_multiplier=args.channel_multiplier, |
|
narrow=args.narrow, |
|
) |
|
|
|
files = sorted(glob.glob(os.path.join(args.indir, "*.*g"))) |
|
for n, file in enumerate(files[:]): |
|
filename = os.path.basename(file) |
|
|
|
im = cv2.imread(file, cv2.IMREAD_COLOR) |
|
if not isinstance(im, np.ndarray): |
|
print(filename, "error") |
|
continue |
|
|
|
|
|
img, orig_faces, enhanced_faces = faceenhancer.process(im) |
|
|
|
im = cv2.resize(im, img.shape[:2][::-1]) |
|
cv2.imwrite(os.path.join(args.outdir, ".".join(filename.split(".")[:-1]) + "_COMP.jpg"), np.hstack((im, img))) |
|
cv2.imwrite(os.path.join(args.outdir, ".".join(filename.split(".")[:-1]) + "_GPEN.jpg"), img) |
|
|
|
for m, (ef, of) in enumerate(zip(enhanced_faces, orig_faces)): |
|
of = cv2.resize(of, ef.shape[:2]) |
|
cv2.imwrite(os.path.join(args.outdir, ".".join(filename.split(".")[:-1]) + "_face%02d" % m + ".jpg"), np.hstack((of, ef))) |
|
|
|
if n % 10 == 0: |
|
print(n, filename) |
|
|