import python

Signed-off-by: Niv Sardi <xaiki@evilgiggle.com>

python/api.py

@@ -0,0 +1,99 @@
+import os
+
+from fastapi import FastAPI, WebSocket
+from YOLOv6.yolov6.core.inferer import Inferer
+
+import cv2
+
+import yaml as YAML
+import json
+import csv
+
+import ssl
+import hashlib
+
+from entity import read_entities
+import imtool
+
+app = FastAPI()
+
+weights = './runs/train/exp27/weights/best_stop_aug_ckpt.pt'
+device = 'cpu'
+yaml = './data.yaml'
+img_size = [640, 640]
+half = False
+conf_thres = 0.5
+iou_thres = 0.45
+classes = None
+agnostic_nms = None
+max_det = 1000
+try:
+    with open(yaml, 'r') as f:
+        classes_data = YAML.safe_load(f.read())
+
+    entities = read_entities('../data/entities.csv')
+
+    certs = {}
+    with os.scandir('../data/certs') as it:
+        for entry in it:
+            bco, ext = entry.name.split('.')
+            if ext == 'cert':
+                try:
+                    cert_dict = ssl._ssl._test_decode_cert(entry.path)
+                    with open(entry.path, 'r') as f:
+                        cert_dict.update({
+                            'fingerprint': hashlib.sha1(
+                                ssl.PEM_cert_to_DER_cert(f.read())
+                            ).hexdigest()
+                        })
+                except Exception as e:
+                    print("Error decoding certificate: {:}".format(e))
+                else:
+                    name = entities[bco].name
+                    certs.update({name: cert_dict})
+
+
+    print(f'loaded {len(certs.keys())} certs, got {len(classes_data["names"])} classes')
+    inferer = Inferer(weights, device, yaml, img_size, half)
+except Exception as e:
+    print('error', e)
+
+
+@app.get("/")
+async def root():
+    return {"message": "API is working"}
+
+@app.websocket("/ws")
+async def websockets_cb(websocket: WebSocket):
+    try:
+        await websocket.accept()
+        while True:
+            data = await websocket.receive_text()
+            img = imtool.read_base64(data)
+            cv2.imwrite("debug.png", img)
+            try:
+                os.remove("debug.txt")
+            except:
+                pass
+
+            inferer.load(img)
+            ret = inferer.infer(conf_thres, iou_thres, classes, agnostic_nms, max_det)
+            print(ret)
+            await websocket.send_text(ret  + '@@@@' + '[%d,%d,%d]'%img.shape)
+    except Exception as e:
+        print("got: ", e)
+
+@app.websocket("/bgws")
+async def send_classes(websocket: WebSocket):
+    await websocket.accept()
+    await websocket.send_text(json.dumps({
+        'classes': classes_data,
+        'certs': certs
+}))
+    await websocket.close()
+
+if __name__ == "__main__":
+    import uvicorn
+    config = uvicorn.Config("api:app", port=5000, log_level="info")
+    server = uvicorn.Server(config)
+    server.run()

python/augment.py

@@ -25,6 +25,48 @@ import pipelines
 
 BATCH_SIZE = 16
 PARALLEL = 20
+MIN_BACKGROUND_SIZE = 500
+
+def process_bg(b):
+
+        imw = cv2.imread(b.path)
+        im, bb = imtool.remove_white(imw)
+        annot = None
+        label = b.path.replace('png', 'txt')
+        if os.path.exists(label):
+            # rewrite label with new coordinates
+            [ww, wh, _] = imw.shape
+            [iw, ih, _] = im.shape
+            es = imtool.read_centroids(label)
+            l = ''
+            for e in es:
+                [i, p, c] = e.values()
+                [x,y,w,h] = [
+                    max((c.x*ww - bb.x)/iw, 0),
+                    max((c.y*wh - bb.y)/ih, 0),
+                    (c.w*ww)/iw,
+                    (c.h*wh)/ih
+                ]
+
+                l += f'{int(i)} {x} {y} {w} {h}\n'
+            annot = l
+
+        if im.shape[0] > args.minbgsize and im.shape[1]> args.minbgsize:
+            return im, annot
+        else:
+            raise Exception(f'droping {b.path} after remove_white => {im.shape}')
+
+def filter_bgs(bgs):
+    ret = []
+    for b in bgs:
+        if b.path.endswith('txt'): continue
+        try:
+            img, annot = process_bg(b)
+        except Exception as e:
+            print(f'drop: {e}')
+            continue
+        ret.append((b, img, annot))
+    return ret
 
 def process(args):
     dest_images_path = os.path.join(args.dest, 'images')
@@ -40,9 +82,13 @@ def process(args):
         reader = csv.DictReader(f)
         db = {e.bco: e for e in [Entity.from_dict(d) for d in reader]}
 
-    background_images = [d for d in os.scandir(args.backgrounds)]
-    assert(len(background_images))
+    background_images = []
+    for d in args.background:
+        background_images.extend(os.scandir(d))
 
+    print(f'filtering {len(background_images)} background images from {args.background}')
+    background_images = filter_bgs(background_images)
+    assert(len(background_images))
     stats = {
         'failed': 0,
         'ok': 0
@@ -69,7 +115,6 @@ def process(args):
             if img.ndim < 3:
                 print(f'very bad dim: {img.ndim}')
 
-            img = imtool.remove_white(img)
             (h, w, c) = img.shape
 
             assert(w > 10)
@@ -95,8 +140,9 @@ def process(args):
             print(f'error loading: {d.path}: {e}')
 
     print(stats)
-    #print(len(logo_alphas), len(logo_images), len(logo_labels))
+
     assert(len(logo_alphas) == len(logo_images))
+    print(f"will process {len(logo_images)} images on {len(background_images)} backgrounds")
 
     # so that we don't get a lot of the same logos on the same page.
     zipped = list(zip(logo_images, logo_alphas))
@@ -117,7 +163,7 @@ def process(args):
 
         batches.append(UnnormalizedBatch(images=a,heatmaps=h))
 
-    bar = ChargingBar('augment', max=(len(batches)**2)/3*len(background_images))
+    bar = ChargingBar(f'augment ({len(logo_images)} logos {len(background_images)} bgs)', max=(len(batches)**2)/3*len(background_images))
     # We use a single, very fast augmenter here to show that batches
     # are only loaded once there is space again in the buffer.
     pipeline = pipelines.HUGE
@@ -137,16 +183,14 @@ def process(args):
         for i, batch_aug in enumerate(batches_aug):
             idx = list(range(len(batch_aug.images_aug)))
             random.shuffle(idx)
-            for j, d in enumerate(background_images):
+            for j, (d, img, annot) in enumerate(background_images):
+                basename = d.name.replace('.png', f'.{i}.{j}')
+                annotations = []
                 try:
-                    img = imtool.remove_white(cv2.imread(d.path))
+                    annotations.append(annot.rstrip())
                 except:
-                    print("couldnt remove white, skipping")
-                    next
-
-                basename = d.name.replace('.png', '') + f'.{i}.{j}'
+                    pass
 
-                anotations = []
                 for k in range(math.floor(len(batch_aug.images_aug)/3)):
                     bar.next()
                     logo_idx = (j+k*4)%len(batch_aug.images_aug)
@@ -165,7 +209,7 @@ def process(args):
                         bb = imtool.mix_alpha(img, logo, alpha[0],
                                               random.random(), random.random())
                         c = bb.to_centroid(img.shape)
-                        anotations.append(c.to_anotation(label))
+                        annotations.append(c.to_annotation(label))
                     except AssertionError as err:
                         print(f'couldnt process {i}, {j}: {err}')
                     except Exception as err:
@@ -175,7 +219,7 @@ def process(args):
                     cv2.imwrite(f'{dest_images_path}/{basename}.png', img)
                     label_path = f"{dest_labels_path}/{basename}.txt"
                     with open(label_path, 'a') as f:
-                        f.write('\n'.join(anotations))
+                        f.write('\n'.join(annotations))
                 except Exception:
                     print(f'couldnt write image {basename}')
 
@@ -186,13 +230,14 @@ def process(args):
 
 if __name__ == '__main__':
     import argparse
-
+    print("✨ augmenting data")
     parser = argparse.ArgumentParser(description='mix backgrounds and logos into augmented data for YOLO')
     parser.add_argument('--logos', metavar='logos', type=str,
                         default=defaults.LOGOS_DATA_PATH,
                         help='dir containing logos')
-    parser.add_argument('--backgrounds', metavar='backgrounds', type=str,
-                        default=defaults.SCREENSHOT_PATH,
+    parser.add_argument('--background', metavar='backgrounds', type=str,
+                        nargs='+',
+                        default=[defaults.SCREENSHOT_PATH, defaults.FISH_PATH],
                         help='dir containing background plates')
     parser.add_argument('--dst', dest='dest', type=str,
                         default=defaults.AUGMENTED_DATA_PATH,
@@ -200,6 +245,7 @@ if __name__ == '__main__':
     parser.add_argument('--parallel', metavar='parallel', type=int,
                         default=PARALLEL,
                         help='number of concurrent jobs')
-
+    parser.add_argument('--min-background-size', dest='minbgsize', type=int,
+                        default=MIN_BACKGROUND_SIZE, help='minimum background size')
     args = parser.parse_args()
     process(args)

python/common/defaults.py

@@ -18,6 +18,7 @@ SQUARES_IMAGES_PATH = D('SQUARES_IMAGES_PATH', f'{SQUARES_DATA_PATH}/images')
 
 DEBUG_PATH = D('DEBUG_PATH', f'{DATA_PATH}/debug')
 DEBUG_SQUARES_PATH = D('DEBUG_SQUARES_PATH', f'{DEBUG_PATH}/squares')
+LOG_PATH = D('LOG_PATH', f'{DATA_PATH}/logs')
 
 LOGOS_DATA_PATH = D('LOGOS_DATA_PATH', f'{DATA_PATH}/logos')
 

python/common/selectors.py

@@ -6,5 +6,5 @@ cls_logo = "*[class*=logo]"
 
 logosbancos = "img[src*=logosbancos]"
 
-entity_http = "p.post-pagina-interior a[target=_blank][href*=http]"
-entity_mailto = "p.post-pagina-interior a[target=_blank][href*=mailto]"
+entity_http = "p.post-pagina-interior [href*=http]"
+entity_mailto = "p.post-pagina-interior [href*=mailto]"

python/crop.py

@@ -1,26 +1,47 @@
 import os
 import argparse
 import imtool
+from progress.bar import ChargingBar
+import concurrent.futures
+
+PARALLEL = 30
+print("🖼 croping augmented data")
 
 parser = argparse.ArgumentParser(description='crop images to train YOLO on squares')
 parser.add_argument('src', metavar='dir', type=str, nargs='+',
                     help='dir containing the images')
 parser.add_argument('--dst', dest='dst', type=str, default='./data/squares',
                     help='dest dir')
+parser.add_argument('--parallel', metavar='parallel', type=int,
+                    default=PARALLEL,
+                    help='number of concurrent jobs')
 
 args = parser.parse_args()
 
+def process(e):
+    if e.name.endswith('.png') and e.is_file():
+        # print(e.name)
+        label = e.path.replace('images', 'labels').replace('.png', '.txt')
+        try:
+            id, boxes = imtool.read_centroids(label)
+            imtool.crop(id, e.path, boxes, args.dst)
+
+        except Exception as err:
+            print(err)
+
 for d in args.src:
-    i = 0
     with os.scandir(d) as it:
-        for e in it:
-            if e.name.endswith('.png') and e.is_file():
-                print(e.name)
-                label = e.path.replace('images', 'labels').replace('.png', '.txt')
-                try:
-                    i+=1
-                    bco, boxes = imtool.read_centroids(label)
-                    imtool.crop(bco, e.path, boxes, args.dst)
+        with concurrent.futures.ThreadPoolExecutor(max_workers = args.parallel) as executor:
+            futures = {executor.submit(process, e): e for e in it}
+            count = len(futures.keys())
+            bar = ChargingBar('crop', max=count)
 
+            print('waiting for futures')
+            for f in concurrent.futures.as_completed(futures):
+                e = futures[f]
+                try:
+                    f.result()
                 except Exception as err:
-                    print(err)
+                    print(f'{a}({e}) generated an exception: {err}')
+                bar.next()
+            bar.finish()

python/get_entities.py

@@ -2,9 +2,11 @@
 import csv
 import requests
 import shutil
+import re
 
 from bs4 import BeautifulSoup
 from progress.bar import ChargingBar
+import concurrent.futures
 
 import web
 from entity import Entity
@@ -17,51 +19,37 @@ soup = BeautifulSoup(page.content, 'html.parser')
 options = soup.find(class_='form-control').find_all('option')
 mkdir.make_dirs([defaults.DATA_PATH, defaults.LOGOS_DATA_PATH])
 
-i = 0
+def get_links(soup):
+    for l in soup.select('.post-pagina-interior'):
+        for a in l.select('a'):
+            if 'href' in a.attrs and a.attrs['href'].startswith('http'):
+                return a.attrs['href']
+
+
 with open(f'{defaults.MAIN_CSV_PATH}.tmp', 'w', newline='') as csvfile:
     writer = csv.writer(csvfile)
     writer.writerow(Entity.row_names())
 
     bar = ChargingBar('get entities', max=len(options))
-    for o in options[1:]:
-        assert(o)
-        def get_bco():
-            (name, bco)= (o.text, o.attrs['value'])
-            page = requests.post(URL, data={'bco': bco})
-            soup = BeautifulSoup(page.content, 'html.parser')
-            img = None
-            try:
-                img = soup.select_one(selectors.logosbancos).attrs['src']
-                img = img.replace('../', 'https://www.bcra.gob.ar/')
-                fn = f"{defaults.LOGOS_DATA_PATH}/{bco}.0.png"
-                web.get_img_logo(img, fn)
-            except AttributeError as err:
-                print(f'couldnt extract image from {img}: {err}')
-                img = None
-
-            a = soup.select_one(selectors.entity_http)
-            try:
-                assert(a)
-                a = a.attrs['href']
-            except AttributeError:
-                a = soup.select_one(selectors.entity_mailto)
-                try:
-                    a = 'http://' + a.attrs['href'].split('@')[1]
-
-                except TypeError:
-                    print('ERROR', a)
-
-            e = Entity(name, id=i, bco=bco, logo=str(img), url=str(a))
-            writer.writerow(e.to_row())
-
-        try:
-            get_bco()
-        except Exception as e:
-            print(f'Error processing: {o.url}')
-
+    def get_bco(o, i):
+        (name, bco)= (o.text, o.attrs['value'])
+
+        page = requests.post(URL, data={'bco': bco}, stream=False)
+        soup = BeautifulSoup(page.content, 'html.parser')
+        img = f'https://www.bcra.gob.ar/Imagenes/logosbancos/{bco}.jpg'
+        e = Entity(name, id=i, bco=bco, logo=str(img), url=str(get_links(soup)))
+        writer.writerow(e.to_row())
         i+=1
+    with concurrent.futures.ThreadPoolExecutor(max_workers = 20) as executor:
+        futures = {executor.submit(get_bco, o, i): o for (i, o) in enumerate(options[1:])}
+        for f in concurrent.futures.as_completed(futures):
+            o = futures[f]
+            try:
+                f.result()
+            except Exception as err:
+                print(f'({o}) generated an exception: {err}')
         bar.next()
     bar.finish()
 
 shutil.move(f'{defaults.MAIN_CSV_PATH}.tmp', defaults.MAIN_CSV_PATH)
-print(f'scrape finished, found {i} entities, dumped to {defaults.MAIN_CSV_PATH}')
+print(f'scrape finished, found {len(options[1:])} entities, dumped to {defaults.MAIN_CSV_PATH}')

python/httpd.py

@@ -0,0 +1,24 @@
+#!/usr/bin/python
+import http.server
+import ssl
+import threading
+
+def launch_httpd(httpd):
+    print(f'launch {httpd.socket}')
+    httpd.serve_forever()
+
+def make_httpd(port):
+    return http.server.HTTPServer(('0.0.0.0', port), http.server.SimpleHTTPRequestHandler)
+
+[httpd, httpsd] = [make_httpd(p) for p in [8080, 8443]]
+
+ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+ctx.load_cert_chain('./cert.pem', keyfile='./privatekey.pem')
+ctx.check_hostname = False
+
+httpsd.socket = ctx.wrap_socket(sock=httpsd.socket, server_side=True)
+
+for h in [httpd, httpsd]:
+    t = threading.Thread(target=launch_httpd, args=(h,))
+    t.start()
+

python/imtool.py

@@ -3,6 +3,7 @@
 import os
 import math
 import cv2
+import base64
 import numpy as np
 from typing import NamedTuple, Tuple, List
 
@@ -32,6 +33,11 @@ class BoundingBox(NamedTuple):
         self = cls(x=d['x'], y=d['y'], w=d['width'], h=d['height'])
         return self
 
+    @classmethod
+    def from_arr(cls, a):
+        self = cls(*a)
+        return self
+
     @property
     def start(self):
         return floor_point(self.x, self.y)
@@ -86,25 +92,33 @@ class Centroid(BoundingBox):
             , w=math.ceil(w*self.w)
             , h=math.ceil(h*self.h))
 
-    def to_anotation(self, id: int):
+    def to_annotation(self, id: int):
         return f'{id} {self.x} {self.y} {self.w} {self.h}'
 
-def read_marker(filename: str, Type: type):
+def read_base64(data):
+    ib = base64.b64decode(data[22:])
+    arr = np.frombuffer(ib, dtype = np.uint8)
+    return cv2.imdecode(arr, flags=cv2.IMREAD_COLOR)
+
+def read_markers(filename: str, Type: type):
     ret = []
-    bco = None
     with open(filename, 'r') as f:
         lines = f.readlines()
         for l in lines:
-            (b, x,y,w,h) = [float(i) for i in l.split(' ')]
-            bco = int(b)
-            ret.append(Type(x,y,w,h))
-    return bco, ret
-
-def read_bounding_boxes(filename: str):
-    return read_marker(filename, BoundingBox)
+            try:
+                (b, x,y,w,h, p) = [float(i) for i in l.split(' ')]
+            except:
+                try:
+                    (b, x,y,w,h) = [float(i) for i in l.split(' ')]
+                except:
+                    continue
+                p = -1
+            ret.append({"class": b, "prob": p, "box": Type(x,y,w,h)})
+    assert(len(ret))
+    return ret
 
 def read_centroids(filename: str):
-    return read_marker(filename, Centroid)
+    return read_markers(filename, Centroid)
 
 def coord_dict_to_point(c: dict):
     return coord_to_point(c['x'], c['y'], c['width'], c['height'])
@@ -138,10 +152,11 @@ def remove_white(img):
     gray = cv2.cvtColor(img, cv2.COLOR_BGRA2GRAY)
     gray = 255*(gray<128)
     coords = cv2.findNonZero(gray)
-    x, y, w, h = cv2.boundingRect(coords) # Find minimum spanning bounding box
-    rect = img[y:y+h, x:x+w] # Crop the image - note we do this on the original image
+    # Find minimum spanning bounding box
+    bb = BoundingBox(*cv2.boundingRect(coords)) 
+    rect = img[bb.y:bb.y+bb.h, bb.x:bb.x+bb.w] # Crop the image - note we do this on the original image
 
-    return rect
+    return rect, bb
 
 
 def mix(a, b, fx, fy):
@@ -157,7 +172,7 @@ def mix_alpha(a, b, ba, fx, fy):
     if (aw*p < bw or ah*p < bh):
         f = min(p*aw/bw, p*ah/bh)
         nw, nh = floor_point(bw*f, bh*f)
-        # print(f'resizing to fit in {aw}x{ah}\t {bw}x{bh}\t=> {nw}x{nh}\tfactor {f}')
+        #print(f'resizing to fit in {aw}x{ah}\t {bw}x{bh}\t=> {nw}x{nh}\tfactor {f}')
         r = cv2.resize(b, (nw, nh), interpolation = cv2.INTER_LINEAR)
         rba = cv2.resize(ba, (nw, nh), interpolation = cv2.INTER_LINEAR)
 
@@ -181,13 +196,15 @@ def _mix_alpha(a, b, ba, fx, fy):
     mask = np.dstack((ba, ba, ba))
 
     a[y:y+bh,x:x+bw] = mat * (1 - mask) + cols * mask
+    #a[y:y+bh,x:x+bw] = cols
 
     return BoundingBox(x, y, bw, bh)
 
-def crop(id, fn, logos: List[Centroid], out = './data/squares', debug_out = './data/debug/'):
+def crop(id, fn, logos: List[Centroid], out = './data/squares'):
     basename = os.path.basename(fn).replace('.png', '')
     img_out = f"{out}/images"
     txt_out = f"{out}/labels"
+    debug_out = f"{defaults.DEBUG_PATH}/{out}"
     mkdir.make_dirs([debug_out, img_out, txt_out])
 
     im = cv2.imread(fn)

python/main.py

@@ -0,0 +1,53 @@
+import csv
+import requests
+import shutil
+
+from bs4 import BeautifulSoup
+from progress.bar import ChargingBar
+
+from entity import Entity
+from common import selectors, defaults, mkdir
+
+URL = 'http://www.bcra.gob.ar/SistemasFinancierosYdePagos/Entidades_financieras.asp'
+page = requests.get(URL)
+soup = BeautifulSoup(page.content, 'html.parser')
+
+options = soup.find(class_='form-control').find_all('option')
+mkdir.make_dirs([defaults.DATA_PATH])
+
+with open(f'{defaults.MAIN_CSV_PATH}.tmp', 'w', newline='') as csvfile:
+    writer = csv.writer(csvfile)
+    writer.writerow(Entity.row_names())
+
+    i = 0
+    bar = ChargingBar('Processing', max=len(options))
+    for o in options[1:]:
+        (name, bco)= (o.text, o.attrs['value'])
+        page = requests.post(URL, data={'bco': bco})
+        soup = BeautifulSoup(page.content, 'html.parser')
+        try:
+            img = soup.select_one(selectors.logosbancos).attrs['src']
+            img = img.replace('../', 'https://www.bcra.gob.ar/')
+        except AttributeError as err:
+            print('img', name, err)
+            img = None
+
+        a = soup.select_one(selectors.entity_http)
+        try:
+            a = a.attrs['href']
+        except AttributeError:
+            a = soup.select_one(selectors.entity_mailto)
+            try:
+                a = 'http://' + a.attrs['href'].split('@')[1]
+
+            except TypeError:
+                print('ERROR', a)
+
+        e = Entity(name, id=i, bco=bco, logo=str(img), url=str(a))
+        writer.writerow(e.to_row())
+        i+=1
+        bar.next()
+    bar.finish()
+
+shutil.move(f'{defaults.MAIN_CSV_PATH}.tmp', defaults.MAIN_CSV_PATH)
+print('scrape finished')

python/markers.py

@@ -1,3 +1,4 @@
+import os
 import cv2
 import argparse
 import imtool
@@ -7,15 +8,37 @@ parser.add_argument('pngs', metavar='img.png', type=str, nargs='+',
                     help='images to debug')
 args = parser.parse_args()
 
-for i in args.pngs:
-    im = cv2.imread(i)
-    label = i.replace('images', 'labels').replace('.png', '.txt').replace('.jpg', '.txt')
-    bco, ccs = imtool.read_centroids(label)
-    bbs = [c.to_bounding_box(im.shape) for c in ccs]
-    for i,b in enumerate(bbs):
-        c = (100, 255*i/len(bbs), 255*(1 - i/len(bbs)))
-        cv2.rectangle(im, b.start, b.end, c, 5)
+if len(args.pngs) and os.path.isdir(args.pngs[0]):
+    args.pngs = [d.path for d in os.scandir(args.pngs[0])]
 
-cv2.imshow('result', im)
-cv2.waitKey(0)
-cv2.destroyAllWindows()
+def process():
+    for i in args.pngs:
+        if i.endswith('txt'): continue
+        im = cv2.imread(i)
+
+        try:
+            assert(im.shape)
+        except AttributeError:
+            print(f'couldnt parse {i}')
+            continue
+
+        label = i.replace('images', 'labels').replace('.png', '.txt').replace('.jpg', '.txt')
+        print(i)
+        try:
+            results = imtool.read_centroids(label)
+        except FileNotFoundError:
+            continue
+        except Exception as e:
+            print(f'error handeling {i}', e)
+            continue
+        bbs = [r["box"].to_bounding_box(im.shape) for r in results]
+        for i,b in enumerate(bbs):
+            print(b)
+            c = (100, 255*i/len(bbs), 255*(1 - i/len(bbs)))
+            cv2.rectangle(im, b.start, b.end, c, 5)
+
+        cv2.imshow('result', im)
+        cv2.waitKey(0)
+        cv2.destroyAllWindows()
+
+process()

python/openfish.py

@@ -43,6 +43,7 @@ def download_all(feed, n_workers=PARALLEL, dest=defaults.FISH_PATH):
 if __name__ == '__main__':
     import argparse
 
+    print("☠ getting extra backgrounds from OpenFish")
     parser = argparse.ArgumentParser(description='screenshot openfish open list')
     parser.add_argument('--parallel', metavar='parallel', type=int,
                         default=PARALLEL,

python/pipelines.py

@@ -13,8 +13,7 @@ sometimes = lambda aug: iaa.Sometimes(0.2, aug)
 HUGE = sometimes(iaa.Sequential(
     [
         # apply the following augmenters to most images
-        iaa.Fliplr(0.5), # horizontally flip 50% of all images
-        iaa.Flipud(0.2), # vertically flip 20% of all images
+        sometimes(iaa.Cartoon()),
         # crop images by -5% to 10% of their height/width
         sometimes(iaa.CropAndPad(
             percent=(-0.05, 0.1),

python/screenshot.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+#
+import math
+
+from selenium import webdriver
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.common.by import By
+
+from common import selectors
+from entity import Entity
+from common import defaults,mkdir
+
+options = webdriver.FirefoxOptions()
+options.add_argument("--headless")
+options.add_argument("--window-size=1920x8000")
+
+def coord_to_point(c):
+    x = math.floor(c['x'] + c['width']/2)
+    y = math.floor(c['y'] + c['height']/2)
+    return f"{x} {y} {math.ceil(c['width'])} {math.ceil(c['height'])}"
+
+driver = webdriver.Firefox(options=options)
+def sc_entity(e: Entity):
+    print(f'screenshoting: {e}')
+    mkdir.make_dirs([
+            defaults.IMAGES_PATH,
+            defaults.LABELS_PATH,
+    ])
+
+    driver.implicitly_wait(10)
+    driver.get(e.url)
+    #driver.save_screenshot(f"{defaults.DATA_PATH}/{e.bco}.png")
+    driver.save_full_page_screenshot(f"{defaults.IMAGES_PATH}/{e.bco}.full.png")
+
+    logos = driver.find_elements(By.CSS_SELECTOR, selectors.img_logo) or []
+    logos.extend(driver.find_elements(By.CSS_SELECTOR, selectors.id_logo) or [])
+    logos.extend(driver.find_elements(By.CSS_SELECTOR, selectors.cls_logo) or [])
+    with open(f"{defaults.LABELS_PATH}/{e.bco}.full.txt", 'w') as f:
+        for i in logos:
+            f.write(f"{e.id} {coord_to_point(i.rect)}\n")
+
+if __name__ == '__main__':
+    sc_entity(Entity.from_dict({'url': 'http://www.bbva.com.ar', 'bco': 'debug'}))

python/split.py

@@ -2,7 +2,6 @@
 import os
 import math
 from common import defaults, mkdir
-
 PATHS = {
     6: {
         'images': lambda dest, d: os.path.join(dest, 'images', d ),
@@ -16,6 +15,8 @@ PATHS = {
 
 if __name__ == '__main__':
     import argparse
+    print("✂ split dataset into train, val and test groups")
+    
     parser = argparse.ArgumentParser(description='splits a yolo dataset between different data partitions')
     parser.add_argument('datapath', metavar='datapath', type=str, 
                         help='csv file', default=defaults.SQUARES_DATA_PATH)
@@ -49,9 +50,14 @@ if __name__ == '__main__':
 
         mkdir.make_dirs([cpi, cpl])
         print( f'{d:6s} [ {p:6d}, {np:6d} ] ({np-p:6d}:{(np-p)/len(images):0.2f} )')
+
+        stats = {'images': 0, 'labels': 0}
         for si in images[p:np]:
+            stats['images'] += 1
             l = image_to_label(si.path)
             os.symlink(os.path.join(rpi, si.name), os.path.join(cpi, si.name))
             if l:
+                stats['labels'] +=1
                 nl = os.path.basename(l)
                 os.symlink(os.path.join(rpl, nl), os.path.join(cpl, nl))
+        print(stats)

python/test.py

@@ -0,0 +1,12 @@
+import re
+import requests
+import logging
+from bs4 import BeautifulSoup
+
+URL = 'http://www.bcra.gob.ar/SistemasFinancierosYdePagos/Entidades_financieras.asp'
+page = requests.post(URL, data={'bco': '00331'}, stream=False)
+soup = BeautifulSoup(page.content, 'html.parser')
+for l in soup.select('.post-pagina-interior'):
+    print(l)
+    for a in l.select('a'):
+        print(a)

python/train.py

@@ -0,0 +1,13 @@
+import yaml
+from entities import read_entities
+
+entities = read_entities()
+
+with open(r'/content/yolov5/data.yaml') as file:
+    # The FullLoader parameter handles the conversion from YAML
+    # scalar values to Python the dictionary format
+    labels_list = yaml.load(file, Loader=yaml.FullLoader)
+
+    label_names = labels_list['names']
+
+print("Number of Classes are {}, whose labels are {} for this Object Detection project".format(num_classes,label_names))

python/vendor.py

@@ -1,17 +1,17 @@
 #!/usr/bin/env python3
-import csv
 import concurrent.futures
 import requests
 
 from progress.bar import ChargingBar
 
-from entity import Entity
+from entity import Entity, read_entities
 from common import defaults,mkdir
 import web
 
 PARALLEL = 20
 
 def do_screenshot(e: Entity):
+    assert(e.url)
     sfn = requests.post('http://puppet:8000/screenshot', json={
         'url': e.url,
         'id': e.id,
@@ -19,18 +19,33 @@ def do_screenshot(e: Entity):
         'logos': f'{defaults.LOGOS_DATA_PATH}/{e.bco}.png'
     })
 
-ACTIONS = [web.get_cert, web.get_logos, do_screenshot]
+def get_entity_logo(e: Entity):
+    fn = f"{defaults.LOGOS_DATA_PATH}/{e.bco}.0.png"
+    web.get_img_logo(e.logo, fn)
 
-def from_csv(fn: str, n_workers = PARALLEL):
-    mkdir.make_dirs([defaults.SCREENSHOT_PATH])
-    with open(fn, newline='') as csvfile:
-        reader = csv.DictReader(csvfile)
-        with concurrent.futures.ThreadPoolExecutor(max_workers = n_workers) as executor:
+def from_csv(args):
+    ACTIONS = []
+    if (args.certs):
+        ACTIONS.append(web.get_cert)
+        mkdir.make_dirs([defaults.CERTS_PATH])
+    if (args.logos):
+        ACTIONS.append(web.get_logos)
+        mkdir.make_dirs([defaults.LOGOS_DATA_PATH])
+    if (args.screenshots):
+        ACTIONS.append(do_screenshot)
+        mkdir.make_dirs([defaults.SCREENSHOT_PATH])
+    if (args.entity_logo):
+        ACTIONS.append(get_entity_logo)
+        mkdir.make_dirs([defaults.LOGOS_DATA_PATH])
+
+    print(ACTIONS)
+    with concurrent.futures.ThreadPoolExecutor(max_workers = args.parallel) as executor:
             futures = {}
-            entities = [Entity.from_dict(d) for d in reader]
-            bar = ChargingBar('vendor', max=len(entities*len(ACTIONS)))
+            entities = read_entities(args.csv)
+            qs = len(entities.keys())*len(ACTIONS)
+            bar = ChargingBar(f'vendor ({qs} jobs)', max=qs)
 
-            for e in entities:
+            for e in entities.values():
                 futures.update({executor.submit(f, e): (e, f) for f in ACTIONS})
             print('waiting for futures')
 
@@ -48,7 +63,7 @@ def from_csv(fn: str, n_workers = PARALLEL):
 
 if __name__ == '__main__':
     import argparse
-
+    print("🌏 getting vendor data")
     parser = argparse.ArgumentParser(description='extract certificates and screenshots websites')
     parser.add_argument('--csv', metavar='csv', type=str,
                         default=defaults.MAIN_CSV_PATH,
@@ -56,6 +71,18 @@ if __name__ == '__main__':
     parser.add_argument('--parallel', metavar='parallel', type=int,
                         default=PARALLEL,
                         help='number of concurrent jobs')
+    parser.add_argument('--logos', metavar='logos', type=bool,
+                        action=argparse.BooleanOptionalAction,
+                        default=True, help='try to get logos')
+    parser.add_argument('--entity-logo', metavar='entity_logo', type=bool,
+                        action=argparse.BooleanOptionalAction,
+                        default=True, help='try to get logos form ENTITY')
+    parser.add_argument('--certs', metavar='certs', type=bool,
+                        action=argparse.BooleanOptionalAction,
+                        default=True, help='try to get certs')
+    parser.add_argument('--screenshots', metavar='screenshots', type=bool,
+                        action=argparse.BooleanOptionalAction,
+                        default=True, help='try to get screenshots')
 
     args = parser.parse_args()
-    from_csv(args.csv)
+    from_csv(args)

python/write_data.py

@@ -1,3 +1,4 @@
+import os
 import csv
 import entity
 import argparse