[ update ]: Cythonize Machine Learning Utilities

.github/workflows/gru_pipeline.yaml

@@ -127,6 +127,15 @@ jobs:
         with:
           name: datasets
 
+      - name: Install Cython and Build ML Module
+        run: |
+          apt-get update && \
+            apt-get install -y gcc python3-dev gnupg
+
+          pip install cython
+          cd training && \
+            python setup.py build_ext --inplace && cd ..
+
       - name: Modeling and Training
         # if: env.match != 'true'
         run: |
@@ -136,7 +145,7 @@ jobs:
           mkdir pickles
           mkdir posttrained
 
-          python training.py \
+          python trainingcli.py \
             --epochs=200     \
             --batchs=32      \
             --sequences=5    \
@@ -568,9 +577,9 @@ jobs:
           sleep 60
           chmod +x endpoints_test.sh && ./endpoints_test.sh
 
-      - name: Set Pipeline Schedule
-        # if: env.match != 'true'
-        run: echo "$(date +'%Y-%m-%d')" > schedulers/gru_schedule.ctl
+      # - name: Set Pipeline Schedule
+      #   if: env.match != 'true'
+      #   run: echo "$(date +'%Y-%m-%d')" > schedulers/gru_schedule.ctl
 
       # - name: Commit changes
       #   if: env.match != 'true'

.github/workflows/lstm_gru_pipeline.yaml

@@ -127,6 +127,15 @@ jobs:
         with:
           name: datasets
 
+      - name: Install Cython and Build ML Module
+        run: |
+          apt-get update && \
+            apt-get install -y gcc python3-dev gnupg
+
+          pip install cython
+          cd training && \
+            python setup.py build_ext --inplace && cd ..
+
       - name: Modeling and Training
         # if: env.match != 'true'
         run: |
@@ -136,15 +145,15 @@ jobs:
           mkdir pickles
           mkdir posttrained
 
-          python training.py \
+          python trainingcli.py \
             --epochs=200     \
             --batchs=32      \
             --sequences=5    \
             --algorithm=LSTM_GRU
 
-      - name: Set Pipeline Schedule
-        # if: env.match != 'true'
-        run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_gru_schedule.ctl
+      # - name: Set Pipeline Schedule
+      #   if: env.match != 'true'
+      #   run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_gru_schedule.ctl
           
       - name: Zip Posttrained, Models, and Pickles
         # if: env.match != 'true'

.github/workflows/lstm_pipeline.yaml

@@ -126,6 +126,15 @@ jobs:
         with:
           name: datasets
 
+      - name: Install Cython and Build ML Module
+        run: |
+          apt-get update && \
+            apt-get install -y gcc python3-dev gnupg
+
+          pip install cython
+          cd training && \
+            python setup.py build_ext --inplace && cd ..
+
       - name: Modeling and Training
         # if: env.match != 'true'
         run: |
@@ -135,15 +144,15 @@ jobs:
           mkdir pickles
           mkdir posttrained
 
-          python training.py \
+          python trainingcli.py \
             --epochs=200     \
             --batchs=32      \
             --sequences=5    \
             --algorithm=LSTM
 
-      - name: Set Pipeline Schedule
-        # if: env.match != 'true'
-        run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_schedule.ctl
+      # - name: Set Pipeline Schedule
+      #   if: env.match != 'true'
+      #   run: echo "$(date +'%Y-%m-%d')" > schedulers/lstm_schedule.ctl
           
       - name: Zip Posttrained, Models, and Pickles
         # if: env.match != 'true'

Makefile

@@ -1,5 +1,56 @@
+# makefile for building and running the application
+
+.PHONY: all cutils ctrain cpostman converter run clean
+
+# default target to build all components
+all: cutils ctrain cpostman
+
+# build Cython extensions in restful/cutils
 cutils:
-	cd restful/cutils && python setup.py build_ext --inplace && cd ../..
+	@echo "Building Cython extensions in restful/cutils..."
+	@cd restful/cutils && python setup.py build_ext --inplace
+
+
+# build Cython extensions in training
+ctrain:
+	@echo "Building Cython extensions in training..."
+	@cd training && python setup.py build_ext --inplace
+
+
+# build Cython extensions in postman
+cpostman:
+	@echo "Building Cython extensions in postman..."
+	@cd postman && python setup.py build_ext --inplace
 
+
+# run the converter script
+converter:
+	@echo "Running converter script..."
+	@python postman/converter.py
+
+
+# run the application with uvicorn
 run:
-	uvicorn app:app --host 0.0.0.0 --port 7860 --reload
+	@echo "Starting the application..."
+	@uvicorn app:app --host 0.0.0.0 --port 7860 --reload
+
+
+# clean up build artifacts
+clean:
+	@echo "Cleaning up build artifacts..."
+	@find . -type f -name "*.so" -delete
+	@find . -type f -name "*.c" -delete
+	@find . -type f -name "*.cpp" -delete
+	@find . -type d -name "__pycache__" -exec rm -r {} +
+
+
+# help message
+help:
+	@echo "Usage:"
+	@echo "  make all       - Build all Cython extensions"
+	@echo "  make cutils    - Build Cython extensions in restful/cutils"
+	@echo "  make ctrain    - Build Cython extensions in training"
+	@echo "  make cpostman  - Build Cython extensions in postman"
+	@echo "  make converter - Run the converter script"
+	@echo "  make run       - Start the application with Uvicorn"
+	@echo "  make clean     - Remove build artifacts"

converter.py → postman/converter.py

@@ -1,5 +1,6 @@
 import json
 
+# Json Processor
 class JSONProcessor:
     def __init__(self, input_file: str, output_file: str) -> None:
         self.input_file:  str = input_file
@@ -30,5 +31,4 @@ def main():
     symbols = processor.extract_symbols()
     processor.save_json(symbols)
 
-
-if __name__ == "__main__": main()
+if __name__ == "__main__": main()

postman/json.c

@@ -0,0 +1 @@
+#error Do not use this file, it is the result of a failed Cython compilation.

postman/json.pyx

@@ -0,0 +1,1000 @@
+from libc.stdlib cimport malloc, free
+from cpython.pycapsule cimport PyCapsule_Import
+from cpython.object cimport PyObject
+from cpython.dict cimport PyDict_GetItemString
+
+
+cdef class JSONProcessor:
+    cdef str input_file
+    cdef str output_file
+    cdef object data
+
+    def __init__(self, str input_file, str output_file):
+        self.input_file = input_file
+        self.output_file = output_file
+        self.data = None
+
+    cpdef void load_json(self):
+        import json
+        with open(self.input_file, 'r') as file:
+            self.data = json.load(file)
+
+    cpdef list extract_symbols(self):
+        if self.data is None:
+            raise ValueError("data not loaded. call load_json() first.")
+        quotes = self.data['finance']['result'][0]['quotes']
+        return [quote['symbol'] for quote in quotes]
+
+    cpdef void save_json(self, list data):
+        import json
+        with open(self.output_file, 'w') as file:
+            json.dump({'symbols': data}, file, indent=4)
+            print(f'saved: {self.output_file}')
+
+
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """
+""" --- """

postman/setup.py

@@ -0,0 +1,9 @@
+from setuptools import setup
+from Cython.Build import cythonize
+
+setup(
+    ext_modules = cythonize(
+      "json.pyx",
+      compiler_directives = {'language_level': "3"}
+    ),
+)

restful/utilities.py

@@ -1,61 +0,0 @@
-import os
-from joblib import load
-from numpy import append, expand_dims
-from pandas import read_json, to_datetime, Timedelta
-
-from tensorflow.keras.models import load_model
-
-
-class Utilities:
-	def __init__(self) -> None:
-		self.model_path = './models'
-		self.posttrained_path = './posttrained'
-		self.scaler_path = './pickles'
-
-	async def cryptocurrency_prediction_utils(self,
-		days: int, sequence_length: int, model_name: str) -> tuple:
-		model_path = os.path.join(self.model_path, f'{model_name}.keras')
-		model = load_model(model_path)
-
-		dataframe_path = os.path.join(self.posttrained_path, f'{model_name}-posttrained.json')
-		dataframe = read_json(dataframe_path)
-		dataframe.set_index('Date', inplace = True)
-
-		minmax_scaler = load(os.path.join(self.scaler_path, f'{model_name}_minmax_scaler.pickle'))
-		standard_scaler = load(os.path.join(self.scaler_path, f'{model_name}_standard_scaler.pickle'))
-
-		lst_seq = dataframe[-sequence_length:].values
-		lst_seq = expand_dims(lst_seq, axis = 0)
-
-		# Predicted
-		predicted_prices = {}
-		last_date = to_datetime(dataframe.index[-1])
-
-		for _ in range(days):
-			predicted_price = model.predict(lst_seq)
-			last_date       = last_date + Timedelta(days = 1)
-
-			predicted_prices[last_date] = minmax_scaler.inverse_transform(predicted_price)
-			predicted_prices[last_date] = standard_scaler.inverse_transform(predicted_prices[last_date])
-
-			lst_seq = append(lst_seq[:, 1:, :], [predicted_price], axis = 1)
-
-		predictions = [
-			{'date': date.strftime('%Y-%m-%d'), 'price': float(price)} \
-				for date, price in predicted_prices.items()
-    ]
-
-		# Actual
-		df_date = dataframe.index[-sequence_length:].values
-		df_date = [to_datetime(date) for date in df_date]
-
-		dataframe[['Close']] = minmax_scaler.inverse_transform(dataframe)
-		dataframe[['Close']] = standard_scaler.inverse_transform(dataframe)
-		df_close = dataframe.iloc[-sequence_length:]['Close'].values
-
-		actuals = [
-			{'date': date.strftime('%Y-%m-%d'), 'price': close} \
-				for date, close in zip(df_date, df_close)
-		]
-        
-		return actuals, predictions

training.py

@@ -1,244 +0,0 @@
-import os
-import json
-import joblib
-import argparse
-import numpy as np
-import pandas as pd
-
-from sklearn.preprocessing import StandardScaler, MinMaxScaler
-
-from tensorflow.keras.models import Sequential
-from tensorflow.keras.layers import GRU, LSTM, Dense, Dropout
-from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
-
-
-
-from warnings import filterwarnings
-filterwarnings('ignore')
-
-class DataProcessor:
-    def __init__(self, datasets_path):
-        self.datasets_path = datasets_path
-        self.datasets = self._get_datasets()
-
-    def _get_datasets(self):
-        return sorted([
-            item for item in os.listdir(self.datasets_path)
-            if os.path.isfile(os.path.join(self.datasets_path, item)) and item.endswith('.csv')
-        ])
-
-    @staticmethod
-    def create_sequences(df, sequence_length):
-        labels, sequences = [], []
-        for i in range(len(df) - sequence_length):
-            seq = df.iloc[i:i + sequence_length].values
-            label = df.iloc[i + sequence_length].values[0]
-            sequences.append(seq)
-            labels.append(label)
-        return np.array(sequences), np.array(labels)
-
-    @staticmethod
-    def preprocess_data(dataframe):
-        for col in dataframe.columns:
-            if dataframe[col].isnull().any():
-                if dataframe[col].dtype == 'object':
-                    dataframe[col].fillna(dataframe[col].mode()[0], inplace = True)
-                else:
-                    dataframe[col].fillna(dataframe[col].mean(), inplace = True)
-        return dataframe
-
-    @staticmethod
-    def scale_data(dataframe, scaler_cls):
-        scaler = scaler_cls()
-        dataframe['Close'] = scaler.fit_transform(dataframe[['Close']])
-        return scaler, dataframe
-
-
-class ModelBuilder:
-    """
-        GRU (Gated Recurrent Units) Model
-    """
-    @staticmethod
-    def gru_model(input_shape):
-        model = Sequential([
-            GRU(50, return_sequences = True, input_shape = input_shape),
-            Dropout(0.2),
-
-            GRU(50, return_sequences = True),
-            Dropout(0.2),
-
-            GRU(50, return_sequences = True),
-            Dropout(0.2),
-
-            GRU(50, return_sequences = False),
-            Dropout(0.2),
-
-            Dense(units = 1)
-        ])
-        model.compile(optimizer = 'nadam', loss = 'mean_squared_error')
-        return model
-
-
-    """
-        LSTM (Long Short-Term Memory) Model
-    """
-    @staticmethod
-    def lstm_model(input_shape):
-        model = Sequential([
-            LSTM(50, return_sequences = True, input_shape = input_shape),
-            Dropout(0.2),
-
-            LSTM(50, return_sequences = True),
-            Dropout(0.2),
-
-            LSTM(50, return_sequences = True),
-            Dropout(0.2),
-
-            LSTM(50, return_sequences = False),
-            Dropout(0.2),
-
-            Dense(units = 1)
-        ])
-        model.compile(optimizer = 'nadam', loss = 'mean_squared_error')
-        return model
-
-
-    """
-        LSTM (Long Short-Term Memory) and
-        GRU (Gated Recurrent Units) Model
-    """
-    @staticmethod
-    def lstm_gru_model(input_shape):
-        model = Sequential([
-            LSTM(50, return_sequences = True, input_shape = input_shape),
-            Dropout(0.2),
-
-            GRU(50, return_sequences = True),
-            Dropout(0.2),
-
-            LSTM(50, return_sequences = True),
-            Dropout(0.2),
-
-            GRU(50, return_sequences = False),
-            Dropout(0.2),
-
-            Dense(units = 1)
-        ])
-        model.compile(optimizer = 'nadam', loss = 'mean_squared_error')
-        return model
-
-
-class Trainer:
-    def __init__(self, model, model_file, sequence_length, epochs, batch_size):
-        self.model = model
-        self.model_file = model_file
-        self.sequence_length = sequence_length
-        self.epochs = epochs
-        self.batch_size = batch_size
-
-    def train(self, X_train, y_train, X_test, y_test):
-        early_stopping = EarlyStopping(monitor = 'val_loss', patience = 5, mode = 'min')
-
-        model_checkpoint = ModelCheckpoint(
-          filepath       = self.model_file,
-          save_best_only = True,
-          monitor        = 'val_loss',
-          mode           = 'min'
-        )
-
-        history = self.model.fit(
-            X_train, y_train,
-            epochs          = self.epochs,
-            batch_size      = self.batch_size,
-            validation_data = (X_test, y_test),
-            callbacks       = [early_stopping, model_checkpoint]
-        )
-
-        return history
-
-
-class PostProcessor:
-    @staticmethod
-    def inverse_transform(scaler, data):
-        return scaler.inverse_transform(data)
-
-    @staticmethod
-    def save_json(filename, data):
-        with open(filename, 'w') as f:
-            json.dump(data, f)
-
-def main(algorithm: str, sequence_length: int, epochs: int, batch_size: int):
-    datasets_path = './datasets'
-    models_path   = './models'
-    posttrained   = './posttrained'
-    pickle_file   = './pickles'
-
-    data_processor = DataProcessor(datasets_path)
-
-    for dataset in data_processor.datasets:
-        print(f"[TRAINING] {dataset.replace('.csv', '')} ")
-
-        dataframe = pd.read_csv(os.path.join(datasets_path, dataset), index_col='Date')[['Close']]
-        model_file = os.path.join(models_path, f"{dataset.replace('.csv', '')}.keras")
-
-        # dataframe = data_processor.preprocess_data(dataframe)
-        dataframe.dropna(inplace = True)
-        standard_scaler, dataframe = data_processor.scale_data(dataframe, StandardScaler)
-        minmax_scaler, dataframe = data_processor.scale_data(dataframe, MinMaxScaler)
-
-        sequences, labels = data_processor.create_sequences(dataframe, sequence_length)
-        input_shape = (sequences.shape[1], sequences.shape[2])
-
-        if algorithm == "GRU":
-            model = ModelBuilder.gru_model(input_shape)
-
-        elif algorithm == "LSTM":
-            model = ModelBuilder.lstm_model(input_shape)
-
-        elif algorithm == "LSTM_GRU":
-            model = ModelBuilder.lstm_gru_model(input_shape)
-
-        else: model = ModelBuilder.lstm_model(input_shape)
-
-        train_size = int(len(sequences) * 0.8)
-        X_train, X_test = sequences[:train_size], sequences[train_size:]
-        y_train, y_test = labels[:train_size], labels[train_size:]
-
-        trainer = Trainer(model, model_file, sequence_length, epochs, batch_size)
-        trainer.train(X_train, y_train, X_test, y_test)
-
-        dataframe_json = {'Date': dataframe.index.tolist(), 'Close': dataframe['Close'].tolist()}
-
-        PostProcessor.save_json(
-          os.path.join(posttrained, f'{dataset.replace(".csv", "")}-posttrained.json'),
-          dataframe_json
-        )
-
-        joblib.dump(minmax_scaler, os.path.join(pickle_file, f'{dataset.replace(".csv", "")}_minmax_scaler.pickle'))
-        joblib.dump(standard_scaler, os.path.join(pickle_file, f'{dataset.replace(".csv", "")}_standard_scaler.pickle'))
-
-        model.load_weights(model_file)
-        model.save(model_file)
-
-        print("\n\n")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description = "Tebakaja Model Trainer")
-
-    parser.add_argument('-a', '--algorithm', type = str, required = True,
-        help = 'select the algorithm to be trained (LSTM, GRU, LSTM_GRU)')
-
-    parser.add_argument('-e', '--epochs', type = int, required = True, help = 'epochs')
-    parser.add_argument('-b', '--batchs', type = int, required = True, help = 'batch length')
-    parser.add_argument('-s', '--sequences', type = int, required = True, help = 'sequences length')
-
-    args = parser.parse_args()
-
-    main(
-        epochs     = args.epochs,
-        batch_size = args.batchs,
-        algorithm  = args.algorithm,
-        sequence_length = args.sequences
-    )
-

training/data_processor.pyx

@@ -0,0 +1,55 @@
+import os
+import numpy as np
+
+from warnings import filterwarnings
+filterwarnings('ignore')
+
+
+""" Get Datasets """
+async def get_datasets(str datasets_path):
+  cdef list items = os.listdir(datasets_path)
+  cdef list csv_files = []
+  cdef str item
+
+  for item in items:
+    if os.path.isfile(os.path.join(datasets_path, item)) and item.endswith('.csv'):
+      csv_files.append(item)
+    
+  return sorted(csv_files)
+
+
+""" Create Sequences """
+async def create_sequences(df, int sequence_length):
+  cdef list labels = []
+  cdef list sequences = []
+  cdef int i
+
+  for i in range(len(df) - sequence_length):
+    seq   = df.iloc[i:i + sequence_length].values
+    label = df.iloc[i + sequence_length].values[0]
+    sequences.append(seq)
+    labels.append(label)
+    
+  return np.array(sequences), np.array(labels)
+
+
+""" Pre-Process Data """
+async def preprocess_data(dataframe):
+  cdef str col
+
+  for col in dataframe.columns:
+    if dataframe[col].isnull().any():
+      if dataframe[col].dtype == 'object':
+        dataframe[col].fillna(dataframe[col].mode()[0], inplace=True)
+      else:
+        dataframe[col].fillna(dataframe[col].mean(), inplace=True)
+
+  return dataframe
+
+
+""" Scale Data """
+async def scale_data(dataframe, scaler_cls):
+  scaler = scaler_cls()
+  dataframe['Close'] = scaler.fit_transform(dataframe[['Close']])
+  return scaler, dataframe
+

training/main.pyx

@@ -0,0 +1,85 @@
+import os
+import joblib
+import argparse
+import pandas as pd
+from sklearn.preprocessing import StandardScaler, MinMaxScaler
+
+
+from training.trainer import train
+from training.post_processor import save_json, inverse_transform
+from training.data_processor import (
+	scale_data,
+	get_datasets,
+	preprocess_data,
+	create_sequences
+)
+
+from training.model_builder import (
+	gru_model,
+	lstm_model,
+	lstm_gru_model
+)
+
+
+from warnings import filterwarnings
+filterwarnings('ignore')
+
+async def main(algorithm: str, sequence_length: int, epochs: int, batch_size: int):
+    datasets_path = './datasets'
+    models_path   = './models'
+    posttrained   = './posttrained'
+    pickle_file   = './pickles'
+
+
+    for dataset in await get_datasets(datasets_path):
+        print(f"[TRAINING] {dataset.replace('.csv', '')} ")
+
+        dataframe = pd.read_csv(os.path.join(datasets_path, dataset), index_col='Date')[['Close']]
+        model_file = os.path.join(models_path, f"{dataset.replace('.csv', '')}.keras")
+
+        # dataframe = preprocess_data(dataframe)
+        dataframe.dropna(inplace = True)
+        standard_scaler, dataframe = await scale_data(dataframe, StandardScaler)
+        minmax_scaler, dataframe = await scale_data(dataframe, MinMaxScaler)
+
+        sequences, labels = await create_sequences(dataframe, sequence_length)
+        input_shape = (sequences.shape[1], sequences.shape[2])
+
+        if algorithm == "GRU":
+            model = await gru_model(input_shape)
+
+        elif algorithm == "LSTM":
+            model = await lstm_model(input_shape)
+
+        elif algorithm == "LSTM_GRU":
+            model = await lstm_gru_model(input_shape)
+
+        else: model = await lstm_model(input_shape)
+
+        train_size = int(len(sequences) * 0.8)
+        X_train, X_test = sequences[:train_size], sequences[train_size:]
+        y_train, y_test = labels[:train_size], labels[train_size:]
+
+        await train({
+			'model': model,
+			'model_file': model_file,
+			'sequence_length': sequence_length,
+			'epochs': epochs,
+			'batch_size': batch_size
+		}, X_train, y_train, X_test, y_test)
+
+        dataframe_json = {'Date': dataframe.index.tolist(), 'Close': dataframe['Close'].tolist()}
+
+        await save_json(
+          os.path.join(posttrained, f'{dataset.replace(".csv", "")}-posttrained.json'),
+          dataframe_json
+        )
+
+        joblib.dump(minmax_scaler, os.path.join(pickle_file, f'{dataset.replace(".csv", "")}_minmax_scaler.pickle'))
+        joblib.dump(standard_scaler, os.path.join(pickle_file, f'{dataset.replace(".csv", "")}_standard_scaler.pickle'))
+
+        model.load_weights(model_file)
+        model.save(model_file)
+
+        print("\n\n")
+

training/model_builder.pyx

@@ -0,0 +1,74 @@
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import GRU, LSTM, Dense, Dropout
+
+from warnings import filterwarnings
+filterwarnings('ignore')
+
+
+""" GRU (Gated Recurrent Units) Model """
+async def gru_model(input_shape):
+  cdef object model = Sequential([
+    GRU(50, return_sequences = True, input_shape = input_shape),
+    Dropout(0.2),
+
+    GRU(50, return_sequences = True),
+    Dropout(0.2),
+
+    GRU(50, return_sequences = True),
+    Dropout(0.2),
+
+    GRU(50, return_sequences = False),
+    Dropout(0.2),
+
+    Dense(units = 1)
+  ])
+
+  model.compile(optimizer = 'nadam', loss = 'mean_squared_error')
+  return model
+
+
+""" LSTM (Long Short-Term Memory) Model """
+async def lstm_model(input_shape):
+  cdef object model = Sequential([
+    LSTM(50, return_sequences = True, input_shape = input_shape),
+    Dropout(0.2),
+
+    LSTM(50, return_sequences = True),
+    Dropout(0.2),
+
+    LSTM(50, return_sequences = True),
+    Dropout(0.2),
+
+    LSTM(50, return_sequences = False),
+    Dropout(0.2),
+
+    Dense(units = 1)
+  ])
+
+  model.compile(optimizer = 'nadam', loss = 'mean_squared_error')
+  return model
+
+
+"""
+  LSTM (Long Short-Term Memory) and
+  GRU (Gated Recurrent Units) Model
+"""
+async def lstm_gru_model(input_shape):
+  cdef object model = Sequential([
+    LSTM(50, return_sequences = True, input_shape = input_shape),
+    Dropout(0.2),
+
+    GRU(50, return_sequences = True),
+    Dropout(0.2),
+
+    LSTM(50, return_sequences = True),
+    Dropout(0.2),
+
+    GRU(50, return_sequences = False),
+    Dropout(0.2),
+
+    Dense(units = 1)
+  ])
+        
+  model.compile(optimizer = 'nadam', loss = 'mean_squared_error')
+  return model

training/post_processor.pyx

@@ -0,0 +1,17 @@
+import json
+from sklearn.preprocessing import MinMaxScaler
+
+from warnings import filterwarnings
+filterwarnings('ignore')
+
+
+""" Inverse Transform """
+async def inverse_transform(object scaler, data):
+	return scaler.inverse_transform(data)
+
+
+""" save json """
+async def save_json(str filename, data):
+	with open(filename, 'w') as f:
+		json.dump(data, f)
+

training/setup.py

@@ -0,0 +1,24 @@
+import numpy as np
+from setuptools import setup
+from Cython.Build import cythonize
+
+setup(
+    ext_modules = cythonize([
+        # Model Builder
+        'model_builder.pyx',
+
+        # Data Processor
+        'data_processor.pyx',
+
+        # Post Processor
+        'post_processor.pyx',
+
+        # Trainer
+        'trainer.pyx',
+
+        # Main
+        'main.pyx'
+    ]),
+
+    include_dirs = [ np.get_include() ]
+)

training/trainer.pyx

@@ -0,0 +1,31 @@
+from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
+
+from warnings import filterwarnings
+filterwarnings('ignore')
+
+
+""" Trainer """
+async def train(dict configuration, X_train, y_train, X_test, y_test):
+	cdef object early_stopping = EarlyStopping(
+		monitor = 'val_loss',
+		patience = 5,
+		mode = 'min'
+	)
+
+	cdef object model_checkpoint = ModelCheckpoint(
+		filepath = configuration['model_file'],
+		save_best_only = True,
+		monitor = 'val_loss',
+		mode = 'min'
+	)
+
+	cdef object history = configuration['model'].fit(
+		X_train, y_train,
+		epochs = configuration['epochs'],
+		batch_size = configuration['batch_size'],
+		validation_data = (X_test, y_test),
+		callbacks = [ early_stopping, model_checkpoint ]
+	)
+
+	return history
+    

trainingcli.py

@@ -0,0 +1,30 @@
+import asyncio
+import argparse
+from training.main import main as training
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description = "Tebakaja Model Trainer")
+
+    parser.add_argument('-a', '--algorithm', type = str, required = True,
+        help = 'select the algorithm to be trained (LSTM, GRU, LSTM_GRU)')
+
+    parser.add_argument('-e', '--epochs', type = int, required = True, help = 'epochs')
+    parser.add_argument('-b', '--batchs', type = int, required = True, help = 'batch length')
+    parser.add_argument('-s', '--sequences', type = int, required = True, help = 'sequences length')
+
+    args = parser.parse_args()
+    event_loop = asyncio.get_event_loop()
+
+    event_loop.run_until_complete(
+        training(
+            epochs     = args.epochs,
+            batch_size = args.batchs,
+            algorithm  = args.algorithm,
+            sequence_length = args.sequences
+        )
+    )
+
+
+if __name__ == "__main__": main()
+