augmentation, first pass

Signed-off-by: Niv Sardi <xaiki@evilgiggle.com>

README.org

@@ -34,7 +34,13 @@ https://github.com/ModelDepot/tfjs-yolo-tiny
 https://github.com/Hyuto/yolov5-tfjs
 
 ** augmentation
-https://github.com/srp-31/Data-Augmentation-for-Object-Detection-YOLO-
+there were a lot of augmentation solutions out there, because it had better
+piplines and multicore support we went with:
+ - https://github.com/aleju/imgaug
+
+but leaving the other here for refs
+ - https://github.com/srp-31/Data-Augmentation-for-Object-Detection-YOLO-
+ - https://github.com/mdbloice/Augmentor
 
 ** proveedores
 http://www.bcra.gov.ar/SistemasFinancierosYdePagos/Proveedores-servicios-de-pago-ofrecen-cuentas-de-pago.asp

python/augment.py

@@ -0,0 +1,111 @@
+import os
+import time
+import math
+import random
+
+from io import BytesIO
+import numpy as np
+from cairosvg import svg2png
+import cv2
+
+import filetype
+from filetype.match import image_matchers
+
+import imgaug as ia
+from imgaug import augmenters as iaa
+from imgaug.augmentables.batches import UnnormalizedBatch
+
+from common import defaults, mkdir
+import imtool
+import pipelines
+
+BATCH_SIZE = 16
+
+mkdir.make_dirs([defaults.AUGMENTED_IMAGES_PATH, defaults.AUGMENTED_LABELS_PATH])
+
+logo_images = []
+background_images = [d for d in os.scandir(defaults.IMAGES_PATH)]
+
+stats = {
+    'failed': 0,
+    'ok': 0
+}
+
+for d in os.scandir(defaults.LOGOS_DATA_PATH):
+    img = None
+    if not d.is_file():
+        stats['failed'] += 1
+        continue
+
+    try:
+        if filetype.match(d.path, matchers=image_matchers):
+            img = cv2.imread(d.path, cv2.IMREAD_UNCHANGED)
+        else:
+            png = svg2png(url=d.path)
+            img = cv2.imdecode(np.asarray(bytearray(png), dtype=np.uint8), cv2.IMREAD_UNCHANGED)
+        stats['ok'] += 1
+
+        (h, w, c) = img.shape
+        if c == 3:
+            img = imtool.add_alpha(img)
+
+        if img.ndim < 3:
+            print(f'very bad dim: {img.ndim}')
+
+        img = imtool.remove_white(img)
+        (h, w, c) = img.shape
+
+        assert(w > 10)
+        assert(h > 10)
+
+        logo_images.append(img)
+    except Exception as e:
+        stats['failed'] += 1
+        print(f'error loading: {d.path}: {e}')
+
+print(stats)
+batches = [UnnormalizedBatch(images=logo_images[i:i+BATCH_SIZE])
+           for i in range(math.floor(len(logo_images)/BATCH_SIZE))]
+
+# We use a single, very fast augmenter here to show that batches
+# are only loaded once there is space again in the buffer.
+pipeline = pipelines.HUGE
+
+def create_generator(lst):
+    for b in lst:
+        print(f"Loading next unaugmented batch...")
+        yield b
+
+batches_generator = create_generator(batches)
+
+with pipeline.pool(processes=-1, seed=1) as pool:
+    batches_aug = pool.imap_batches(batches_generator, output_buffer_size=5)
+
+    print(f"Requesting next augmented batch...")
+    for i, batch_aug in enumerate(batches_aug):
+        idx = list(range(len(batch_aug.images_aug)))
+        random.shuffle(idx)
+        for j, d in enumerate(background_images):
+            img = imtool.remove_white(cv2.imread(d.path))
+            basename = d.name.replace('.png', '') + f'.{i}.{j}'
+
+            anotations = []
+            for k in range(math.floor(len(batch_aug.images_aug)/3)):
+                logo = batch_aug.images_aug[(j+k)%len(batch_aug.images_aug)]
+                try:
+                    img, bb, (w, h) = imtool.mix(img, logo, random.random(), random.random())
+                    anotations.append(f'0 {bb.x/w} {bb.y/h} {bb.w/w} {bb.h/h}')
+                except AssertionError:
+                    print(f'couldnt process {i}, {j}')
+
+            try:
+                cv2.imwrite(f'{defaults.AUGMENTED_IMAGES_PATH}/{basename}.png', img)
+                label_path = f"{defaults.AUGMENTED_LABELS_PATH}/{basename}.txt"
+                with open(label_path, 'a') as f:
+                    f.write('\n'.join(anotations))
+            except Exception:
+                print(f'couldnt write image {basename}')
+
+        if i < len(batches)-1:
+            print("Requesting next augmented batch...")
+

python/imtool.py

@@ -3,6 +3,7 @@
 import os
 import math
 import cv2
+import numpy as np
 from typing import NamedTuple
 
 from entity import Entity
@@ -38,16 +39,26 @@ class Centroid(BoundingBox):
 
 def read_bounding_boxes(filename):
     boxes = []
+    bco = None
     with open(filename, 'r') as f:
         lines = f.readlines()
         for l in lines:
-            (bco, x,y,w,h) = [float(i) for i in l.split(' ')]
+            (b, x,y,w,h) = [float(i) for i in l.split(' ')]
+            bco = b
             if x < 0 or y < 0 or w < 10 or h < 10:
-                print(f"dropping logo, it has inconsistent size: {w}x{h}+{x}x{y}")
+                print(f"dropping logo, it has inconsistent size: {w}x{h}@{x}x{y}")
                 continue
             boxes.append(BoundingBox(x,y,w,h))
     return bco, boxes
 
+def coord_dict_to_point(c):
+    return coord_to_point(c['x'], c['y'], c['width'], c['heigh'])
+
+def coord_to_point(cx, cy, cw, ch):
+    x = math.floor(cx + cw/2)
+    y = math.floor(cy + ch/2)
+    return f"{x} {y} {math.ceil(cw)} {math.ceil(ch)}"
+
 def floor_point(x, y):
     return (math.floor(x), math.floor(y))
 
@@ -64,6 +75,39 @@ def cut_logo(im, l):
     (x, y, w, h) = floor_logo(l)
     return im[x:w, y:h]
 
+def add_alpha(img):
+    b, g, r = cv2.split(img)
+    a = np.ones(b.shape, dtype=b.dtype) * 50
+    return cv2.merge((b,g,r,a))
+
+def remove_white(img):
+    gray = cv2.cvtColor(img, cv2.COLOR_BGRA2GRAY)
+    gray = 255*(gray<128)
+    coords = cv2.findNonZero(gray)
+    x, y, w, h = cv2.boundingRect(coords) # Find minimum spanning bounding box
+    rect = img[y:y+h, x:x+w] # Crop the image - note we do this on the original image
+
+    return rect
+
+def mix(a, b, fx, fy):
+    (ah, aw, ac) = a.shape
+    (bh, bw, bc) = b.shape
+
+    assert(aw > bw)
+    assert(ah > bh)
+
+    x = math.floor(fx*(aw - bw))
+    y = math.floor(fy*(ah - bh))
+
+    mat = a[y:y+bh,x:x+bw]
+    cols = b[:, :, :3]
+    alpha = b[:, :, 3]/255
+    mask = np.dstack((alpha, alpha, alpha))
+
+    a[y:y+bh,x:x+bw] = mat * (1 - mask) + cols * mask
+
+    return a, BoundingBox(x, y, bw, bh), (aw, ah)
+
 def crop(id, fn, logos):
     basename = os.path.basename(fn).replace('.png', '')
     img_out = f"./data/squares/images"
@@ -85,7 +129,7 @@ def crop(id, fn, logos):
     for x in range(tx):
         for y in range(ty):
             color = (0,x*(255/tx),y*(255/ty))
-
+            logo_color = (255, 0, 0)
 
             if tx < 2:
                 xs = 0
@@ -104,6 +148,10 @@ def crop(id, fn, logos):
             rim = cv2.rectangle(rim, start, end, color, 10)
             li = []
             for l in logos:
+                rim = cv2.rectangle(rim,
+                                    floor_point(l.x, l.y),
+                                    floor_point(l.x + l.w, l.y + l.h),
+                                    logo_color, 5)
                 def intersect():
                     six = l.x - f.x
                     siy = l.y - f.y
@@ -135,8 +183,6 @@ def crop(id, fn, logos):
                 if p:
                     li.append(p)
 
-            c = (255, 0, 0)
-
             nim = im[start[1]:end[1], start[0]:end[0]]
             rnim = rim[start[1]:end[1], start[0]:end[0]]
             img_name =f"{img_out}/{basename}-x{x}y{y}.jpg"
@@ -152,7 +198,7 @@ def crop(id, fn, logos):
                         dim = cv2.rectangle(rnim,
                                            floor_point(cx - p.w/2, cy - p.h/2),
                                            floor_point(cx + p.w/2, cy + p.h/2),
-                                           c,
+                                           logo_color,
                                            5)
 
                         a = f"{int(id)} {cx/TILE_SIZE} {cy/TILE_SIZE} {p.w/TILE_SIZE} {p.h/TILE_SIZE}\n"
@@ -160,7 +206,7 @@ def crop(id, fn, logos):
                         print(a)
                         cv2.imwrite(f'{debug_out}/{basename}{x}{y}.debug.png', dim)
 
-    cv2.imwrite(f'{debug_out}/{basename}.debug.png', im)
+    cv2.imwrite(f'{debug_out}/{basename}.debug.png', rim)
 
 if __name__ == '__main__':
     i = 0

python/pipelines.py

@@ -0,0 +1,79 @@
+import imgaug as ia
+from imgaug import augmenters as iaa
+
+# Sometimes(0.5, ...) applies the given augmenter in 50% of all cases,
+# e.g. Sometimes(0.5, GaussianBlur(0.3)) would blur roughly every second image.
+sometimes = lambda aug: iaa.Sometimes(0.1, aug)
+
+# Define our sequence of augmentation steps that will be applied to every image
+# All augmenters with per_channel=0.5 will sample one value _per image_
+# in 50% of all cases. In all other cases they will sample new values
+# _per channel_.
+
+HUGE = sometimes(iaa.Sequential(
+    [
+        # apply the following augmenters to most images
+        iaa.Fliplr(0.5), # horizontally flip 50% of all images
+        iaa.Flipud(0.2), # vertically flip 20% of all images
+        # crop images by -5% to 10% of their height/width
+        sometimes(iaa.CropAndPad(
+            percent=(-0.05, 0.1),
+            pad_mode=ia.ALL,
+            pad_cval=(0, 255)
+        )),
+        sometimes(iaa.Affine(
+            scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis
+            translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)
+            rotate=(-45, 45), # rotate by -45 to +45 degrees
+            shear=(-16, 16), # shear by -16 to +16 degrees
+            order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)
+            cval=(0, 255), # if mode is constant, use a cval between 0 and 255
+            mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)
+        )),
+        # execute 0 to 5 of the following (less important) augmenters per image
+        # don't execute all of them, as that would often be way too strong
+        iaa.SomeOf((0, 5),
+            [
+                sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation
+                iaa.OneOf([
+                    iaa.GaussianBlur((0, 3.0)), # blur images with a sigma between 0 and 3.0
+                    iaa.AverageBlur(k=(2, 7)), # blur image using local means with kernel sizes between 2 and 7
+                    iaa.MedianBlur(k=(3, 11)), # blur image using local medians with kernel sizes between 2 and 7
+                ]),
+                iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images
+                iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images
+                # search either for all edges or for directed edges,
+                # blend the result with the original image using a blobby mask
+                iaa.SimplexNoiseAlpha(iaa.OneOf([
+                    iaa.EdgeDetect(alpha=(0.5, 1.0)),
+                    iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)),
+                ])),
+                iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5), # add gaussian noise to images
+                iaa.OneOf([
+                    iaa.Dropout((0.01, 0.1), per_channel=0.5), # randomly remove up to 10% of the pixels
+                    iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),
+                ]),
+                iaa.Invert(0.05, per_channel=True), # invert color channels
+                iaa.Add((-10, 10), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)
+                iaa.AddToHueAndSaturation((-20, 20)), # change hue and saturation
+                # either change the brightness of the whole image (sometimes
+                # per channel) or change the brightness of subareas
+                iaa.OneOf([
+                    iaa.Multiply((0.5, 1.5), per_channel=0.5),
+                    iaa.FrequencyNoiseAlpha(
+                        exponent=(-4, 0),
+                        first=iaa.Multiply((0.5, 1.5), per_channel=True),
+                        second=iaa.LinearContrast((0.5, 2.0))
+                    )
+                ]),
+                iaa.LinearContrast((0.5, 2.0), per_channel=0.5), # improve or worsen the contrast
+                iaa.Grayscale(alpha=(0.0, 1.0)),
+                sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)
+                sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))), # sometimes move parts of the image around
+                sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))
+            ],
+            random_order=True
+        )
+    ],
+    random_order=True
+))

python/requirements.txt

@@ -1,4 +1,11 @@
-bs4==0.0.1
+beautifulsoup4==4.11.1
+CairoSVG==2.5.2
+filetype==1.1.0
+imgaug==0.4.0
+inotify==0.2.10
+numpy==1.23.2
+opencv_python==4.6.0.66
 progress==1.6
-inotify
-requests
+PyYAML==6.0
+requests==2.27.1
+selenium==4.4.3

python/screenshot.py

@@ -10,16 +10,12 @@ from selenium.webdriver.common.by import By
 from common import selectors
 from entity import Entity
 from common import defaults,mkdir
+from imtool import coord_dict_to_point
 
 options = webdriver.FirefoxOptions()
 options.add_argument("--headless")
 options.add_argument("--window-size=1920x8000")
 
-def coord_to_point(c):
-    x = math.floor(c['x'] + c['width']/2)
-    y = math.floor(c['y'] + c['height']/2)
-    return f"{x} {y} {math.ceil(c['width'])} {math.ceil(c['height'])}"
-
 driver = webdriver.Firefox(options=options)
 def sc_entity(e: Entity):
     print(f'screenshoting: {e}')
@@ -38,7 +34,7 @@ def sc_entity(e: Entity):
     logos.extend(driver.find_elements(By.CSS_SELECTOR, selectors.cls_logo) or [])
     with open(f"{defaults.LABELS_PATH}/{e.bco}.full.txt", 'w') as f:
         for i in logos:
-            f.write(f"{e.id} {coord_to_point(i.rect)}\n")
+            f.write(f"{e.id} {coord_dict_to_point(i.rect)}\n")
 
 if __name__ == '__main__':
     sc_entity(Entity.from_dict({'url': 'http://www.bbva.com.ar', 'bco': 'debug'}))

python/web.py

@@ -17,14 +17,14 @@ def get_page(e: Entity):
 
 def get_cert(e: Entity):
     ssl_url = e.url.split("/")[2]
-    mkdir.make_dirs(defaults.CERTS_PATH)
+    mkdir.make_dirs([defaults.CERTS_PATH])
     try:
         cert = ssl.get_server_certificate((ssl_url, 443), ca_certs=None)
         fn = f"{defaults.CERTS_PATH}/{e.bco}.cert"
         with open(fn, 'w') as f:
             f.write(cert)
     except Exception as err:
-        with open(f"{fn}.error.log", 'w+') as f:
+        with open(f"{defaults.DATA_PATH}/{e.bco}.error.log", 'w+') as f:
             f.write(str(err))
     return fn
 
@@ -40,7 +40,7 @@ def get_logos(e: Entity, page):
     logos.extend(soup.select(selectors.id_logo))
     logos.extend(soup.select(selectors.cls_logo))
 
-    mkdir.make_dirs(defaults.LOGOS_DATA_PATH)
+    mkdir.make_dirs([defaults.LOGOS_DATA_PATH])
 
     i = 0
     lfn = []