move from complicated multi-container to simpler design with a shell script

Dockerfile.python

@@ -2,8 +2,12 @@ FROM docker.io/python:3-slim-buster
 MAINTAINER Niv Sardi <x@filtra.me>
 WORKDIR /app
 
-COPY python ./src
-RUN pip install -r ./src/requirements.txt
-RUN apt update && apt install -y libglib2.0-0 libgl1 && rm -rf /var/cache/apt
+RUN apt update && apt install -y libcairo2 libglib2.0-0 libgl1 && rm -rf /var/cache/apt
+COPY python/requirements.txt ./python/requirements.txt
+RUN pip install -r ./python/requirements.txt
 
-CMD python3 ./src/watcher.py
+COPY run.sh ./run
+RUN chmod +x run
+COPY python ./python
+
+CMD env PATH=$PATH:/usr/local/bin python3 ./python/watcher.py

README.org

@@ -18,7 +18,8 @@ The process is pretty simple:
 * running
 #+begin_src sh
   # build the training dataset
-  docker-compose up --build --remove-orphans
+  docker-compose up --build --remove-orphans -d
+  docker-compose exec python ./run
 
   # run the training on your machine or collab
   # https://colab.research.google.com/drive/10R7uwVJJ1R1k6oTjbkkhxPDka7COK-WE

docker-compose.yaml

@@ -1,50 +1,75 @@
 version: "3.9"  # optional since v1.27.0
 services:
-  crawler:
+  python:
     build:
       dockerfile: Dockerfile.python
       context: .
-    command: "python3 src/main.py"
-    volumes:
-      - "./data:/app/data:z"
-
-  puppet:
-    build:
-      dockerfile: Dockerfile.deno
-      context: .
-    links:
-      - browserless
     environment:
-      BROWSERLESS_HOST: browserless
-      BROWSERLESS_PORT: 3000
-      DEBUG: "puppet"
-    depends_on:
-      - "browserless"
-    # command: "sh -c 'while echo deno; do sleep 3h; done'" # debug
-    command: "deno run --allow-net --allow-env --allow-read --allow-write src/index.ts"
-    volumes:
-       - "./deno:/app/src:z" # for debugging
-       - "./data:/app/data:z"
-    #restart: unless-stopped:600
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 600s
-        window: 300s
-
-  cutter:
-    build:
-      dockerfile: Dockerfile.python
-      context: .
+       GECKO_HOST: geckodriver
+       GECKO_PORT: 4444
     depends_on:
-      - "puppet"
+      - "geckodriver"
+    links:
+      - "geckodriver"
     volumes:
-      - "./python:/app/src:z" # for debugging
+      - "./python:/app/python:z" # for debugging
       - "./data:/app/data:z"
 
-  browserless:
-     image: docker.io/zenika/alpine-chrome
-     entrypoint: ["sh", "-c", "while true; do chromium-browser --headless --use-gl=swiftshader --disable-software-rasterizer --disable-dev-shm-usage --no-sandbox --remote-debugging-address=0.0.0.0 --remote-debugging-port=3000; sleep 2; done"]
-     ports:
-       - "3000:3000"
+  geckodriver:
+    image: docker.io/instrumentisto/geckodriver
+    entrypoint: ["sh", "-c", "while true; do geckodriver --binary=/opt/firefox/firefox --log warn --port 4444 --host 0.0.0.0; sleep 2; done"]
+    ports: # this is not required but nice for local debug
+      - "4444:4444"
+  # crawler:
+  #   build:
+  #     dockerfile: Dockerfile.python
+  #     context: .
+  #   command: "sh -c 'while true; do python3 src/get_entities.py; touch data/entities.csv; sleep 24h; done'"
+  #   volumes:
+  #     - "./data:/app/data:z"
+
+  # cutter:
+  #   build:
+  #     dockerfile: Dockerfile.python
+  #     context: .
+  #   environment:
+  #      GECKO_HOST: geckodriver
+  #      GECKO_PORT: 4444
+  #   depends_on:
+  #     - "geckodriver"
+  #   links:
+  #     - "geckodriver"
+  #   volumes:
+  #     - "./python:/app/src:z" # for debugging
+  #     - "./data:/app/data:z"
+
+  # browserless:
+  #    image: docker.io/zenika/alpine-chrome
+  #    entrypoint: ["sh", "-c", "while true; do chromium-browser --headless --use-gl=swiftshader --disable-software-rasterizer --disable-dev-shm-usage --no-sandbox --remote-debugging-address=0.0.0.0 --remote-debugging-port=3000; sleep 2; done"]
+  #    ports:
+  #      - "3000:3000"
+
+  # puppet:
+  #   build:
+  #     dockerfile: Dockerfile.deno
+  #     context: .
+  #   links:
+  #     - browserless
+  #   environment:
+  #     BROWSERLESS_HOST: browserless
+  #     BROWSERLESS_PORT: 3000
+  #     DEBUG: "puppet"
+  #   depends_on:
+  #     - "browserless"
+  #   #command: "sh -c 'while echo deno; do sleep 3h; done'" # debug
+  #   command: "deno run --allow-net --allow-env --allow-read --allow-write src/index.ts"
+  #   volumes:
+  #      - "./deno:/app/src:z" # for debugging
+  #      - "./data:/app/data:z"
+  #   #restart: unless-stopped:600
+  #   deploy:
+  #     restart_policy:
+  #       condition: any
+  #       delay: 600s
+  #       window: 300s
 

python/{main.py → get_entities.py}

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 import csv
 import requests
 import shutil
@@ -14,43 +15,50 @@ page = requests.get(URL)
 soup = BeautifulSoup(page.content, 'html.parser')
 
 options = soup.find(class_='form-control').find_all('option')
-mkdir.make_dirs([defaults.DATA_PATH])
+mkdir.make_dirs([defaults.DATA_PATH, defaults.LOGOS_DATA_PATH])
 
+i = 0
 with open(f'{defaults.MAIN_CSV_PATH}.tmp', 'w', newline='') as csvfile:
     writer = csv.writer(csvfile)
     writer.writerow(Entity.row_names())
 
-    i = 0
     bar = ChargingBar('Processing', max=len(options))
     for o in options[1:]:
-        (name, bco)= (o.text, o.attrs['value'])
-        page = requests.post(URL, data={'bco': bco})
-        soup = BeautifulSoup(page.content, 'html.parser')
-        try:
-            img = soup.select_one(selectors.logosbancos).attrs['src']
-            img = img.replace('../', 'https://www.bcra.gob.ar/')
-            fn = f"{defaults.LOGOS_DATA_PATH}/{bco}.0.png"
-            web.get_img_logo(img, fn)
-        except AttributeError as err:
-            print('img', name, err)
-            img = None
-
-        a = soup.select_one(selectors.entity_http)
-        try:
-            a = a.attrs['href']
-        except AttributeError:
-            a = soup.select_one(selectors.entity_mailto)
+        def get_bco():
+            (name, bco)= (o.text, o.attrs['value'])
+            page = requests.post(URL, data={'bco': bco})
+            soup = BeautifulSoup(page.content, 'html.parser')
+            try:
+                img = soup.select_one(selectors.logosbancos).attrs['src']
+                img = img.replace('../', 'https://www.bcra.gob.ar/')
+                fn = f"{defaults.LOGOS_DATA_PATH}/{bco}.0.png"
+                web.get_img_logo(img, fn)
+            except AttributeError as err:
+                print('img', name, err)
+                img = None
+
+            a = soup.select_one(selectors.entity_http)
             try:
-                a = 'http://' + a.attrs['href'].split('@')[1]
+                a = a.attrs['href']
+            except AttributeError:
+                a = soup.select_one(selectors.entity_mailto)
+                try:
+                    a = 'http://' + a.attrs['href'].split('@')[1]
 
-            except TypeError:
-                print('ERROR', a)
+                except TypeError:
+                    print('ERROR', a)
+
+            e = Entity(name, id=i, bco=bco, logo=str(img), url=str(a))
+            writer.writerow(e.to_row())
+
+        try:
+            get_bco()
+        except Exception as e:
+            print(f'Error processing: {e}')
 
-        e = Entity(name, id=i, bco=bco, logo=str(img), url=str(a))
-        writer.writerow(e.to_row())
         i+=1
         bar.next()
     bar.finish()
 
 shutil.move(f'{defaults.MAIN_CSV_PATH}.tmp', defaults.MAIN_CSV_PATH)
-print('scrape finished')
+print(f'scrape finished, found {i} entities, dumped to {defaults.MAIN_CSV_PATH}')

run.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+PY=python3
+echo "🏛 fetching entities"
+${PY} ./python/get_entities.py
+echo "🌏 getting vendor data"
+${PY} ./python/vendor.py --parallel $(cat /proc/cpuinfo | grep processor | wc -l)
+echo "✨ augmenting data"
+${PY} ./python/augment.py
+echo "🖼 croping augmented data"
+${PY} ./python/crop.py ./data/augmented/images
+echo "TODO: 🧠 train model"