import random import numpy as np import matplotlib matplotlib.use('agg') import matplotlib.pyplot as plt import math from skimage.transform import warp, AffineTransform import cv2 from scipy import misc from skimage.transform import rotate from PIL import Image from PIL import ImageOps from skimage.transform import resize from skimage import transform from skimage.transform import SimilarityTransform, AffineTransform import random from configuration import DatasetName class ImageUtility: def crop_and_save(self, _image, _label, file_name, num_of_landmarks, dataset_name): try: '''crop data: we add a small margin to the images''' xy_points, x_points, y_points = self.create_landmarks(landmarks=_label, scale_factor_x=1, scale_factor_y=1) # self.print_image_arr(str(x_points[0]), _image, x_points, y_points) img_arr, points_arr = self.cropImg(_image, x_points, y_points, no_padding=False) # img_arr = output_img # points_arr = t_label '''resize image to 224*224''' resized_img = resize(img_arr, (224, 224, 3), anti_aliasing=True) dims = img_arr.shape height = dims[0] width = dims[1] scale_factor_y = 224 / height scale_factor_x = 224 / width '''rescale and retrieve landmarks''' landmark_arr_xy, landmark_arr_x, landmark_arr_y = \ self.create_landmarks(landmarks=points_arr, scale_factor_x=scale_factor_x, scale_factor_y=scale_factor_y) min_b = 0.0 max_b = 224 if not(min(landmark_arr_x) < min_b or min(landmark_arr_y) < min_b or max(landmark_arr_x) > max_b or max(landmark_arr_y) > max_b): # self.print_image_arr(str(landmark_arr_x[0]), resized_img, landmark_arr_x, landmark_arr_y) im = Image.fromarray((resized_img * 255).astype(np.uint8)) im.save(str(file_name) + '.jpg') pnt_file = open(str(file_name) + ".pts", "w") pre_txt = ["version: 1 \n", "n_points: 68 \n", "{ \n"] pnt_file.writelines(pre_txt) points_txt = "" for i in range(0, len(landmark_arr_xy), 2): points_txt += str(landmark_arr_xy[i]) + " " + str(landmark_arr_xy[i + 1]) + "\n" pnt_file.writelines(points_txt) pnt_file.write("} \n") pnt_file.close() except Exception as e: print(e) def random_rotate(self, _image, _label, file_name, num_of_landmarks, dataset_name): try: xy_points, x_points, y_points = self.create_landmarks(landmarks=_label, scale_factor_x=1, scale_factor_y=1) # self.print_image_arr(str(xy_points[8]), _image, x_points, y_points) _image, _label = self.cropImg_2time(_image, x_points, y_points) _image = self.__noisy(_image) scale = (np.random.uniform(0.8, 1.0), np.random.uniform(0.8, 1.0)) # scale = (1, 1) rot = np.random.uniform(-1 * 0.55, 0.55) translation = (0, 0) shear = 0 tform = AffineTransform( scale=scale, # , rotation=rot, translation=translation, shear=np.deg2rad(shear) ) output_img = transform.warp(_image, tform.inverse, mode='symmetric') sx, sy = scale t_matrix = np.array([ [sx * math.cos(rot), -sy * math.sin(rot + shear), 0], [sx * math.sin(rot), sy * math.cos(rot + shear), 0], [0, 0, 1] ]) landmark_arr_xy, landmark_arr_x, landmark_arr_y = self.create_landmarks(_label, 1, 1) label = np.array(landmark_arr_x + landmark_arr_y).reshape([2, num_of_landmarks]) marging = np.ones([1, num_of_landmarks]) label = np.concatenate((label, marging), axis=0) label_t = np.dot(t_matrix, label) lbl_flat = np.delete(label_t, 2, axis=0).reshape([2*num_of_landmarks]) t_label = self.__reorder(lbl_flat, num_of_landmarks) '''crop data: we add a small margin to the images''' xy_points, x_points, y_points = self.create_landmarks(landmarks=t_label, scale_factor_x=1, scale_factor_y=1) img_arr, points_arr = self.cropImg(output_img, x_points, y_points, no_padding=False) # img_arr = output_img # points_arr = t_label '''resize image to 224*224''' resized_img = resize(img_arr, (224, 224, 3), anti_aliasing=True) dims = img_arr.shape height = dims[0] width = dims[1] scale_factor_y = 224 / height scale_factor_x = 224 / width '''rescale and retrieve landmarks''' landmark_arr_xy, landmark_arr_x, landmark_arr_y = \ self.create_landmarks(landmarks=points_arr, scale_factor_x=scale_factor_x, scale_factor_y=scale_factor_y) min_b = 0.0 max_b = 224 if dataset_name == DatasetName.cofw: min_b = 5.0 max_b = 214 if not(min(landmark_arr_x) < 0 or min(landmark_arr_y) < min_b or max(landmark_arr_x) > 224 or max(landmark_arr_y) > max_b): # self.print_image_arr(str(landmark_arr_x[0]), resized_img, landmark_arr_x, landmark_arr_y) im = Image.fromarray((resized_img * 255).astype(np.uint8)) im.save(str(file_name) + '.jpg') pnt_file = open(str(file_name) + ".pts", "w") pre_txt = ["version: 1 \n", "n_points: 68 \n", "{ \n"] pnt_file.writelines(pre_txt) points_txt = "" for i in range(0, len(landmark_arr_xy), 2): points_txt += str(landmark_arr_xy[i]) + " " + str(landmark_arr_xy[i + 1]) + "\n" pnt_file.writelines(points_txt) pnt_file.write("} \n") pnt_file.close() return t_label, output_img except Exception as e: print(e) return None, None def random_rotate_m(self, _image, _label_img, file_name): rot = random.uniform(-80.9, 80.9) output_img = rotate(_image, rot, resize=True) output_img_lbl = rotate(_label_img, rot, resize=True) im = Image.fromarray((output_img * 255).astype(np.uint8)) im_lbl = Image.fromarray((output_img_lbl * 255).astype(np.uint8)) im_m = ImageOps.mirror(im) im_lbl_m = ImageOps.mirror(im_lbl) im.save(str(file_name)+'.jpg') # im_lbl.save(str(file_name)+'_lbl.jpg') im_m.save(str(file_name) + '_m.jpg') # im_lbl_m.save(str(file_name) + '_m_lbl.jpg') im_lbl_ar = np.array(im_lbl) im_lbl_m_ar = np.array(im_lbl_m) self.__save_label(im_lbl_ar, file_name, np.array(im)) self.__save_label(im_lbl_m_ar, file_name+"_m", np.array(im_m)) def __save_label(self, im_lbl_ar, file_name, img_arr): im_lbl_point = [] for i in range(im_lbl_ar.shape[0]): for j in range(im_lbl_ar.shape[1]): if im_lbl_ar[i, j] != 0: im_lbl_point.append(j) im_lbl_point.append(i) pnt_file = open(str(file_name)+".pts", "w") pre_txt = ["version: 1 \n", "n_points: 68 \n", "{ \n"] pnt_file.writelines(pre_txt) points_txt = "" for i in range(0, len(im_lbl_point), 2): points_txt += str(im_lbl_point[i]) + " " + str(im_lbl_point[i+1]) + "\n" pnt_file.writelines(points_txt) pnt_file.write("} \n") pnt_file.close() '''crop data: we add a small margin to the images''' xy_points, x_points, y_points = self.create_landmarks(landmarks=im_lbl_point, scale_factor_x=1, scale_factor_y=1) img_arr, points_arr = self.cropImg(img_arr, x_points, y_points) '''resize image to 224*224''' resized_img = resize(img_arr, (224, 224, 3), anti_aliasing=True) dims = img_arr.shape height = dims[0] width = dims[1] scale_factor_y = 224 / height scale_factor_x = 224 / width '''rescale and retrieve landmarks''' landmark_arr_xy, landmark_arr_x, landmark_arr_y = \ self.create_landmarks(landmarks=points_arr, scale_factor_x=scale_factor_x, scale_factor_y=scale_factor_y) im = Image.fromarray((resized_img * 255).astype(np.uint8)) im.save(str(im_lbl_point[0])+'.jpg') # self.print_image_arr(im_lbl_point[0], resized_img, landmark_arr_x, landmark_arr_y) def augment(self, _image, _label, num_of_landmarks): # face = misc.face(gray=True) # # rotate_face = ndimage.rotate(_image, 45) # self.print_image_arr(_label[0], rotate_face, [],[]) # hue_img = tf.image.random_hue(_image, max_delta=0.1) # max_delta must be in the interval [0, 0.5]. # sat_img = tf.image.random_saturation(hue_img, lower=0.0, upper=3.0) # # sat_img = K.eval(sat_img) # _image = self.__noisy(_image) shear = 0 # rot = 0.0 '''this scale has problem''' # scale = (random.uniform(0.8, 1.00), random.uniform(0.8, 1.00)) scale = (1, 1) rot = np.random.uniform(-1 * 0.008, 0.008) tform = AffineTransform(scale=scale, rotation=rot, shear=shear, translation=(0, 0)) output_img = warp(_image, tform.inverse, output_shape=(_image.shape[0], _image.shape[1])) sx, sy = scale t_matrix = np.array([ [sx * math.cos(rot), -sy * math.sin(rot + shear), 0], [sx * math.sin(rot), sy * math.cos(rot + shear), 0], [0, 0, 1] ]) landmark_arr_xy, landmark_arr_x, landmark_arr_y = self.create_landmarks(_label, 1, 1) label = np.array(landmark_arr_x + landmark_arr_y).reshape([2, num_of_landmarks]) marging = np.ones([1, num_of_landmarks]) label = np.concatenate((label, marging), axis=0) label_t = np.dot(t_matrix, label) lbl_flat = np.delete(label_t, 2, axis=0).reshape([num_of_landmarks*2]) t_label = self.__reorder(lbl_flat, num_of_landmarks) return t_label, output_img def __noisy(self, image): noise_typ = random.randint(0, 5) # if True or noise_typ == 0 :#"gauss": # row, col, ch = image.shape # mean = 0 # var = 0.001 # sigma = var ** 0.1 # gauss = np.random.normal(mean, sigma, (row, col, ch)) # gauss = gauss.reshape(row, col, ch) # noisy = image + gauss # return noisy if 1 <= noise_typ <= 2:# "s&p": row, col, ch = image.shape s_vs_p = 0.5 amount = 0.04 out = np.copy(image) # Salt mode num_salt = np.ceil(amount * image.size * s_vs_p) coords = [np.random.randint(0, i - 1, int(num_salt)) for i in image.shape] out[coords] = 1 # Pepper mode num_pepper = np.ceil(amount * image.size * (1. - s_vs_p)) coords = [np.random.randint(0, i - 1, int(num_pepper)) for i in image.shape] out[coords] = 0 return out # elif 5 <=noise_typ <= 7: #"speckle": # row, col, ch = image.shape # gauss = np.random.randn(row, col, ch) # gauss = gauss.reshape(row, col, ch) # noisy = image + image * gauss # return noisy else: return image def __reorder(self, input_arr, num_of_landmarks): out_arr = [] for i in range(num_of_landmarks): out_arr.append(input_arr[i]) k = num_of_landmarks + i out_arr.append(input_arr[k]) return np.array(out_arr) def print_image_arr_heat(self, k, image): plt.figure() plt.imshow(image) implot = plt.imshow(image) plt.axis('off') plt.savefig('heat' + str(k) + '.png', bbox_inches='tight') plt.clf() def print_image_arr(self, k, image, landmarks_x, landmarks_y): plt.figure() plt.imshow(image) implot = plt.imshow(image) plt.scatter(x=landmarks_x[:], y=landmarks_y[:], c='black', s=20) plt.scatter(x=landmarks_x[:], y=landmarks_y[:], c='white', s=15) plt.axis('off') plt.savefig('sss' + str(k) + '.png', bbox_inches='tight') # plt.show() plt.clf() def create_landmarks_from_normalized_original_img(self, img, landmarks, width, height, x_center, y_center, x1, y1, scale_x, scale_y): # landmarks_splited = _landmarks.split(';') landmark_arr_xy = [] landmark_arr_x = [] landmark_arr_y = [] for j in range(0, len(landmarks), 2): x = ((x_center - float(landmarks[j]) * width)*scale_x) + x1 y = ((y_center - float(landmarks[j + 1]) * height)*scale_y) + y1 landmark_arr_xy.append(x) landmark_arr_xy.append(y) landmark_arr_x.append(x) landmark_arr_y.append(y) img = cv2.circle(img, (int(x), int(y)), 2, (255, 14, 74), 2) img = cv2.circle(img, (int(x), int(y)), 1, (0, 255, 255), 1) return landmark_arr_xy, landmark_arr_x, landmark_arr_y, img def create_landmarks_from_normalized(self, landmarks, width, height, x_center, y_center): # landmarks_splited = _landmarks.split(';') landmark_arr_xy = [] landmark_arr_x = [] landmark_arr_y = [] for j in range(0, len(landmarks), 2): x = x_center - float(landmarks[j]) * width y = y_center - float(landmarks[j + 1]) * height landmark_arr_xy.append(x) landmark_arr_xy.append(y) # [ x1, y1, x2,y2 ] landmark_arr_x.append(x) # [x1, x2] landmark_arr_y.append(y) # [y1, y2] return landmark_arr_xy, landmark_arr_x, landmark_arr_y def create_landmarks(self, landmarks, scale_factor_x, scale_factor_y): # landmarks_splited = _landmarks.split(';') landmark_arr_xy = [] landmark_arr_x = [] landmark_arr_y = [] for j in range(0, len(landmarks), 2): x = float(landmarks[j]) * scale_factor_x y = float(landmarks[j + 1]) * scale_factor_y landmark_arr_xy.append(x) landmark_arr_xy.append(y) # [ x1, y1, x2,y2 ] landmark_arr_x.append(x) # [x1, x2] landmark_arr_y.append(y) # [y1, y2] return landmark_arr_xy, landmark_arr_x, landmark_arr_y def create_landmarks_aflw(self, landmarks, scale_factor_x, scale_factor_y): # landmarks_splited = _landmarks.split(';') landmark_arr_xy = [] landmark_arr_x = [] landmark_arr_y = [] for j in range(0, len(landmarks), 2): if landmarks[j][0] == 1: x = float(landmarks[j][1]) * scale_factor_x y = float(landmarks[j][2]) * scale_factor_y landmark_arr_xy.append(x) landmark_arr_xy.append(y) # [ x1, y1, x2,y2 ] landmark_arr_x.append(x) # [x1, x2] landmark_arr_y.append(y) # [y1, y2] return landmark_arr_xy, landmark_arr_x, landmark_arr_y def random_augmentation(self, lbl, img, number_of_landmark): # a = random.randint(0, 2) # if a == 0: # img, lbl = self.__add_margin(img, img.shape[0], lbl) '''this function has problem!!!''' # img, lbl = self.__add_margin(img, img.shape[0], lbl) # else: # img, lbl = self.__negative_crop(img, lbl) # i = random.randint(0, 2) # if i == 0: # img, lbl = self.__rotate(img, lbl, 90, img.shape[0], img.shape[1]) # elif i == 1: # img, lbl = self.__rotate(img, lbl, 180, img.shape[0], img.shape[1]) # else: # img, lbl = self.__rotate(img, lbl, 270, img.shape[0], img.shape[1]) # k = random.randint(0, 3) # if k > 0: # img = self.__change_color(img) # img = self.__noisy(img) lbl = np.reshape(lbl, [number_of_landmark*2]) return lbl, img def cropImg_2time(self, img, x_s, y_s): min_x = max(0, int(min(x_s) - 100)) max_x = int(max(x_s) + 100) min_y = max(0, int(min(y_s) - 100)) max_y = int(max(y_s) + 100) crop = img[min_y:max_y, min_x:max_x] new_x_s = [] new_y_s = [] new_xy_s = [] for i in range(len(x_s)): new_x_s.append(x_s[i] - min_x) new_y_s.append(y_s[i] - min_y) new_xy_s.append(x_s[i] - min_x) new_xy_s.append(y_s[i] - min_y) return crop, new_xy_s def cropImg(self, img, x_s, y_s, no_padding=False): margin1 = random.randint(0, 10) margin2 = random.randint(0, 10) margin3 = random.randint(0, 10) margin4 = random.randint(0, 10) if no_padding: min_x = max(0, int(min(x_s))) max_x = int(max(x_s)) min_y = max(0, int(min(y_s))) max_y = int(max(y_s)) else: min_x = max(0, int(min(x_s) - margin1)) max_x = int(max(x_s) + margin2) min_y = max(0, int(min(y_s) - margin3)) max_y = int(max(y_s) + margin4) crop = img[min_y:max_y, min_x:max_x] new_x_s = [] new_y_s = [] new_xy_s = [] for i in range(len(x_s)): new_x_s.append(x_s[i] - min_x) new_y_s.append(y_s[i] - min_y) new_xy_s.append(x_s[i] - min_x) new_xy_s.append(y_s[i] - min_y) # imgpr.print_image_arr(k, crop, new_x_s, new_y_s) # imgpr.print_image_arr_2(i, img, x_s, y_s, [min_x, max_x], [min_y, max_y]) return crop, new_xy_s def __negative_crop(self, img, landmarks): landmark_arr_xy, x_s, y_s = self.create_landmarks(landmarks, 1, 1) min_x = img.shape[0] // random.randint(5, 15) max_x = img.shape[0] - (img.shape[0] // random.randint(15, 20)) min_y = img.shape[0] // random.randint(5, 15) max_y = img.shape[0] - (img.shape[0] // random.randint(15, 20)) crop = img[min_y:max_y, min_x:max_x] new_x_s = [] new_y_s = [] new_xy_s = [] for i in range(len(x_s)): new_x_s.append(x_s[i] - min_x) new_y_s.append(y_s[i] - min_y) new_xy_s.append(x_s[i] - min_x) new_xy_s.append(y_s[i] - min_y) # imgpr.print_image_arr(crop.shape[0], crop, new_x_s, new_y_s) # imgpr.print_image_arr_2(crop.shape[0], crop, x_s, y_s, [min_x, max_x], [min_y, max_y]) return crop, new_xy_s def __add_margin(self, img, img_w, lbl): marging_width = img_w // random.randint(15, 20) direction = random.randint(0, 4) if direction == 1: margings = np.random.random([img_w, int(marging_width), 3]) img = np.concatenate((img, margings), axis=1) if direction == 2: margings_1 = np.random.random([img_w, int(marging_width), 3]) img = np.concatenate((img, margings_1), axis=1) marging_width_1 = img_w // random.randint(15, 20) margings_2 = np.random.random([int(marging_width_1), img_w + int(marging_width), 3]) img = np.concatenate((img, margings_2), axis=0) if direction == 3: # need chane labels margings_1 = np.random.random([img_w, int(marging_width), 3]) img = np.concatenate((margings_1, img), axis=1) lbl = self.__transfer_lbl(int(marging_width), lbl, [1, 0]) marging_width_1 = img_w // random.randint(15, 20) margings_2 = np.random.random([int(marging_width_1), img_w + int(marging_width), 3]) img = np.concatenate((margings_2, img), axis=0) lbl = self.__transfer_lbl(int(marging_width_1), lbl, [0, 1]) if direction == 4: # need chane labels margings_1 = np.random.random([img_w, int(marging_width), 3]) img = np.concatenate((margings_1, img), axis=1) lbl = self.__transfer_lbl(int(marging_width), lbl, [1, 0]) img_w1 = img_w + int(marging_width) marging_width_1 = img_w // random.randint(15, 20) margings_2 = np.random.random([int(marging_width_1), img_w1, 3]) img = np.concatenate((margings_2, img), axis=0) lbl = self.__transfer_lbl(int(marging_width_1), lbl, [0, 1]) img_w2 = img_w + int(marging_width_1) marging_width_1 = img_w // random.randint(15, 20) margings_1 = np.random.random([img_w2, int(marging_width_1), 3]) img = np.concatenate((img, margings_1), axis=1) marging_width_1 = img_w // random.randint(15, 20) margings_2 = np.random.random([int(marging_width_1), img.shape[1], 3]) img = np.concatenate((img, margings_2), axis=0) return img, lbl def __void_image(self, img, img_w, ): marging_width = int(img_w / random.randint(7, 16)) direction = random.randint(0, 1) direction = 0 if direction == 0: np.delete(img, 100, 1) # img[:, 0:marging_width, :] = 0 elif direction == 1: img[img_w - marging_width:img_w, :, :] = 0 if direction == 2: img[:, img_w - marging_width:img_w, :] = 0 return img def __change_color(self, img): # color_arr = np.random.random([img.shape[0], img.shape[1]]) color_arr = np.zeros([img.shape[0], img.shape[1]]) axis = random.randint(0, 4) if axis == 0: # red img_mono = img[:, :, 0] new_img = np.stack([img_mono, color_arr, color_arr], axis=2) elif axis == 1: # green img_mono = img[:, :, 1] new_img = np.stack([color_arr, img_mono, color_arr], axis=2) elif axis == 2: # blue img_mono = img[:, :, 1] new_img = np.stack([color_arr, img_mono, color_arr], axis=2) elif axis == 3: # gray scale img_mono = img[:, :, 0] new_img = np.stack([img_mono, img_mono, img_mono], axis=2) else: # random noise color_arr = np.random.random([img.shape[0], img.shape[1]]) img_mono = img[:, :, 0] new_img = np.stack([img_mono, img_mono, color_arr], axis=2) return new_img def __rotate_origin_only(self, xy_arr, radians, xs, ys): """Only rotate a point around the origin (0, 0).""" rotated = [] for xy in xy_arr: x, y = xy xx = x * math.cos(radians) + y * math.sin(radians) yy = -x * math.sin(radians) + y * math.cos(radians) rotated.append([xx + xs, yy + ys]) return np.array(rotated) def __rotate(self, img, landmark_old, degree, img_w, img_h, num_of_landmarks): landmark_old = np.reshape(landmark_old, [num_of_landmarks, 2]) theta = math.radians(degree) if degree == 90: landmark = self.__rotate_origin_only(landmark_old, theta, 0, img_h) return np.rot90(img, 3, axes=(-2, 0)), landmark elif degree == 180: landmark = self.__rotate_origin_only(landmark_old, theta, img_h, img_w) return np.rot90(img, 2, axes=(-2, 0)), landmark elif degree == 270: landmark = self.__rotate_origin_only(landmark_old, theta, img_w, 0) return np.rot90(img, 1, axes=(-2, 0)), landmark def __transfer_lbl(self, marging_width_1, lbl, axis_arr): new_lbl = [] for i in range(0, len(lbl), 2): new_lbl.append(lbl[i] + marging_width_1 * axis_arr[0]) new_lbl.append(lbl[i + 1] + marging_width_1 * axis_arr[1]) return np.array(new_lbl)