Repository: mindsdb/lightwood

Issue:
Make directory for storing custom modules
As a replacement for the current `JsonAI.imports` mechanism, Lightwood should create a directory where all custom ML blocks are stored (potentially following the same structure as core Lightwood). This directory would be turned into a Python module so that at runtime Lightwood automatically imports any custom block that is discovered, streamlining the process.



Discussion:

Resolving patch:
diff --git a/lightwood/api/json_ai.py b/lightwood/api/json_ai.py
--- a/lightwood/api/json_ai.py
+++ b/lightwood/api/json_ai.py
@@ -302,7 +302,6 @@ def generate_json_ai(
         explainer=None,
         features=features,
         outputs=outputs,
-        imports=None,
         problem_definition=problem_definition,
         identifiers=type_information.identifiers,
         timeseries_transformer=None,
@@ -322,36 +321,6 @@ def add_implicit_values(json_ai: JsonAI) -> JsonAI:
     problem_definition = json_ai.problem_definition
     tss = problem_definition.timeseries_settings
 
-    imports = [
-        'from lightwood.mixer import Neural', 'from lightwood.mixer import LightGBM',
-        'from lightwood.mixer import LightGBMArray', 'from lightwood.mixer import SkTime',
-        'from lightwood.mixer import Unit', 'from lightwood.mixer import Regression',
-        'from lightwood.ensemble import BestOf', 'from lightwood.data import cleaner',
-        'from lightwood.data import transform_timeseries, timeseries_analyzer', 'from lightwood.data import splitter',
-        'from lightwood.analysis import model_analyzer, explain',
-        'from sklearn.metrics import r2_score, balanced_accuracy_score, accuracy_score', 'import pandas as pd',
-        'from lightwood.helpers.seed import seed', 'from lightwood.helpers.log import log', 'import lightwood',
-        'from lightwood.api import *', 'from lightwood.mixer import BaseMixer',
-        'from lightwood.encoder import BaseEncoder, __ts_encoders__',
-        'from lightwood.encoder import Array, Binary, Categorical, Date, Datetime, TimeSeries, Float, Image, Integer, Quantity, Rich_Text, Short_Text, Tags', # noqa
-        'from lightwood.ensemble import BaseEnsemble', 'from typing import Dict, List',
-        'from lightwood.helpers.parallelism import mut_method_call',
-        'from lightwood.data.encoded_ds import ConcatedEncodedDs', 'from lightwood import ProblemDefinition']
-
-    if json_ai.imports is None:
-        json_ai.imports = imports
-    else:
-        json_ai.imports.extend(imports)
-
-    for feature in [list(json_ai.outputs.values())[0], *json_ai.features.values()]:
-        encoder_import = feature.encoder['module']
-        if "." in encoder_import:
-            continue
-        imports.append(f"from lightwood.encoder import {encoder_import}")
-
-    if tss.use_previous_target:
-        imports.append('from lightwood.encoder import ArrayEncoder')
-
     # Add implicit arguments
     # @TODO: Consider removing once we have a proper editor in studio
     mixers = json_ai.outputs[json_ai.problem_definition.target].mixers
@@ -691,11 +660,47 @@ def code_from_json_ai(json_ai: JsonAI) -> str:
 """
     predict_proba_body = align(predict_proba_body, 2)
 
-    imports = "\n".join(json_ai.imports)
+    imports = """
+import lightwood
+from lightwood.analysis import *
+from lightwood.api import *
+from lightwood.data import *
+from lightwood.encoder import *
+from lightwood.ensemble import *
+from lightwood.helpers.device import *
+from lightwood.helpers.general import *
+from lightwood.helpers.log import *
+from lightwood.helpers.numeric import *
+from lightwood.helpers.parallelism import *
+from lightwood.helpers.seed import *
+from lightwood.helpers.text import *
+from lightwood.helpers.torch import *
+from lightwood.mixer import *
+from lightwood.encoder import __ts_encoders__
+import pandas as pd
+from typing import Dict, List
+import os
+import importlib.machinery
+import os
+import importlib.machinery
+from types import ModuleType
+import sys"""
+
+    import_external_dir = """
+for import_dir in [os.path.expanduser('~/lightwood_modules'), '/etc/lightwood_modules']:
+    if os.path.exists(import_dir) and os.access(import_dir, os.R_OK):
+        for file_name in list(os.walk(import_dir))[0][2]:
+            print(file_name)
+            mod_name = file_name.rstrip('.py')
+            loader = importlib.machinery.SourceFileLoader(mod_name,
+                                                          os.path.join(import_dir, file_name))
+            module = ModuleType(loader.name)
+            loader.exec_module(module)
+            exec(f'{mod_name} = module')
+"""
     predictor_code = f"""
 {imports}
-from lightwood.api import PredictorInterface
-
+{import_external_dir}
 
 class Predictor(PredictorInterface):
     target: str
diff --git a/lightwood/api/types.py b/lightwood/api/types.py
--- a/lightwood/api/types.py
+++ b/lightwood/api/types.py
@@ -430,7 +430,6 @@ class JsonAI:
     :param splitter: The Splitter object is the method in which the input data is split into training/validation/testing data.
     :param analyzer: The Analyzer object is used to evaluate how well a model performed on the predictive task.
     :param explainer: The Explainer object deploys explainability tools of interest on a model to indicate how well a model generalizes its predictions.
-    :param imports: A list of custom packages, indicated through a str import statement, that a user can call.
     :param timeseries_transformer:
     :param timeseries_analyzer:
     :param accuracy_functions: A list of performance metrics used to evaluate the best models.
@@ -444,7 +443,6 @@ class JsonAI:
     splitter: Optional[object] = None
     analyzer: Optional[object] = None
     explainer: Optional[object] = None
-    imports: Optional[List[str]] = None
     timeseries_transformer: Optional[object] = None
     timeseries_analyzer: Optional[object] = None
     accuracy_functions: Optional[List[str]] = None
@@ -463,7 +461,6 @@ def from_dict(obj: Dict):
         splitter = obj.get("splitter", None)
         analyzer = obj.get("analyzer", None)
         explainer = obj.get("explainer", None)
-        imports = obj.get("imports", None)
         timeseries_transformer = obj.get("timeseries_transformer", None)
         timeseries_analyzer = obj.get("timeseries_analyzer", None)
         accuracy_functions = obj.get("accuracy_functions", None)
@@ -478,7 +475,6 @@ def from_dict(obj: Dict):
             splitter=splitter,
             analyzer=analyzer,
             explainer=explainer,
-            imports=imports,
             timeseries_transformer=timeseries_transformer,
             timeseries_analyzer=timeseries_analyzer,
             accuracy_functions=accuracy_functions,
diff --git a/lightwood/helpers/factory.py b/lightwood/helpers/factory.py
deleted file mode 100644
--- a/lightwood/helpers/factory.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from functools import partial
-
-# Factories are syntactic sugar but can introduce bugs, we should *only* use them for the public APIs
-
-
-class Factory:
-    self.generate_class = None
-
-    def __init__(self, **kwargs):
-        pass
-
-    def generate(self) -> self.generate_class:
-        return self.generate_class(**kwargs)
-
-
-def gen_factory_func(func: Callable, **kwargs) -> Callable:
-    return partial(func, **kwargs)


Test patch:
diff --git a/tests/integration/advanced/test_custom_modules.py b/tests/integration/advanced/test_custom_modules.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/advanced/test_custom_modules.py
@@ -0,0 +1,57 @@
+from lightwood.api.high_level import json_ai_from_problem, code_from_json_ai, predictor_from_code
+from lightwood.api.types import JsonAI, ProblemDefinition
+import unittest
+from mindsdb_datasources import FileDS
+import os
+import shutil
+
+
+test_err_message = 'This ! Is ! A ! Testing ! Error !'
+
+
+def create_custom_module():
+    mdir = os.path.expanduser('~/lightwood_modules')
+    mpath = os.path.join(mdir, 'custom_cleaners.py')
+    try:
+        shutil.rmtree(mpath)
+    except Exception:
+        pass
+
+    try:
+        os.mkdir(mdir)
+    except Exception:
+        pass
+
+    with open(mpath, 'w') as fp:
+        fp.write(f'def throwing_cleaner(): raise Exception("{test_err_message}")')
+
+
+class TestBasic(unittest.TestCase):
+    def test_0_add_throwing_cleaner(self):
+        create_custom_module()
+
+        # Create base json ai
+        df = FileDS('tests/data/hdi.csv').df.iloc[0:400]
+        json_ai = json_ai_from_problem(df, ProblemDefinition.from_dict({'target': 'Development Index', 'time_aim': 20}))
+
+        # modify it
+        json_ai_dump = json_ai.to_dict()
+        json_ai_dump['cleaner'] = {
+            'module': 'custom_cleaners.throwing_cleaner',
+            'args': {}
+        }
+        print(json_ai_dump)
+
+        json_ai = JsonAI.from_dict(json_ai_dump)
+
+        # create a predictor from it
+        code = code_from_json_ai(json_ai)
+        predictor = predictor_from_code(code)
+        try:
+            predictor.learn(df)
+        except Exception as e:
+            assert str(e) == test_err_message
+            return
+
+        raise Exception('Predictor did not contain modified function!')
+