Repository: mindsdb/lightwood

Issue:
PyOD as analysis block
[Reference](https://github.com/yzhao062/pyod)


Discussion:
Pending: build scaffolding in JsonAI so that the right data arrives here.

Resolving patch:
diff --git a/lightwood/analysis/__init__.py b/lightwood/analysis/__init__.py
--- a/lightwood/analysis/__init__.py
+++ b/lightwood/analysis/__init__.py
@@ -1,8 +1,8 @@
-# Base
+# Phases
 from lightwood.analysis.analyze import model_analyzer
 from lightwood.analysis.explain import explain
 
-# Blocks
+# Base blocks
 from lightwood.analysis.base import BaseAnalysisBlock
 from lightwood.analysis.nc.calibrate import ICP
 from lightwood.analysis.helpers.acc_stats import AccStats
@@ -10,11 +10,14 @@
 from lightwood.analysis.nn_conf.temp_scale import TempScaler
 from lightwood.analysis.helpers.feature_importance import PermutationFeatureImportance
 
+# Blocks with extra requirements
 try:
     from lightwood.analysis.helpers.shap import ShapleyValues
+    from lightwood.analysis.helpers.pyod import PyOD
 except Exception:
     ShapleyValues = None
+    PyOD = None
 
 
-__all__ = ['model_analyzer', 'explain', 'BaseAnalysisBlock', 'TempScaler',
+__all__ = ['model_analyzer', 'explain', 'BaseAnalysisBlock', 'TempScaler', 'PyOD',
            'ICP', 'AccStats', 'ConfStats', 'PermutationFeatureImportance', 'ShapleyValues']
diff --git a/lightwood/analysis/helpers/pyod.py b/lightwood/analysis/helpers/pyod.py
new file mode 100644
--- /dev/null
+++ b/lightwood/analysis/helpers/pyod.py
@@ -0,0 +1,98 @@
+from copy import deepcopy
+from types import SimpleNamespace
+from typing import Dict, Optional, Tuple
+
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import OrdinalEncoder
+
+from pyod.models.lof import LOF
+from pyod.models.pca import PCA
+from pyod.models.hbos import HBOS
+from pyod.models.suod import SUOD
+
+from lightwood.analysis.base import BaseAnalysisBlock
+from lightwood.helpers.log import log
+from lightwood.helpers.parallelism import get_nr_procs
+
+
+class PyOD(BaseAnalysisBlock):
+    """
+    Wrapper analysis block for the 'PyOD' anomaly detection library.
+
+    For now, the following techniques are supported:
+      - SUOD: Large-scale unsupervised heterogeneous outlier detection
+    """  # noqa
+
+    def __init__(self, contamination=0.1, deps: Optional[Tuple] = ...):
+        super().__init__(deps=deps)
+        self.target = None
+        self.input_cols = []
+        self.exceptions = ['__make_predictions']
+        self.ordinal_encoders = dict()
+        if contamination > 0.5:
+            log.warning("Contamination higher than maximum possible value, setting at 0.5...")
+        self.contamination = min(contamination, 0.5)
+
+    def analyze(self, info: Dict[str, object], **kwargs) -> Dict[str, object]:
+        log.info('Preparing to compute anomaly detection with PyOD')
+        ns = SimpleNamespace(**kwargs)
+        self.target = ns.target
+        detector_list = [LOF(n_neighbors=10, contamination=self.contamination),
+                         LOF(n_neighbors=30, contamination=self.contamination),
+                         HBOS(contamination=self.contamination),
+                         PCA(contamination=self.contamination)]
+
+        df = deepcopy(ns.train_data.data_frame)
+        n_jobs = get_nr_procs(df)
+        clf = SUOD(base_estimators=detector_list,
+                   contamination=self.contamination,
+                   n_jobs=n_jobs,
+                   combination='average',
+                   verbose=False)
+
+        if ns.tss.is_timeseries:
+            for gcol in ns.tss.group_by:
+                self.ordinal_encoders[gcol] = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1)
+                self.ordinal_encoders[gcol].fit(df[gcol].values.reshape(-1, 1))
+            for i in range(1, ns.tss.horizon):
+                self.exceptions.append(f'{self.target}_timestep_{i}')
+
+            self.exceptions.append(ns.tss.order_by)
+            df = self._preprocess_ts_df(df, ns)
+
+        for col in df.columns:
+            if col != self.target and '__mdb' not in col and col not in self.exceptions:
+                self.input_cols.append(col)
+
+        df = df.loc[:, self.input_cols].dropna()
+        clf.fit(df.values)
+        info['pyod_explainer'] = clf
+        return info
+
+    def explain(self,
+                row_insights: pd.DataFrame,
+                global_insights: Dict[str, object],
+                **kwargs
+                ) -> Tuple[pd.DataFrame, Dict[str, object]]:
+        log.info('Performing anomaly detection with PyOD...')
+        ns = SimpleNamespace(**kwargs)
+        pyod_explainer = ns.analysis.get('pyod_explainer', None)
+        if pyod_explainer is None:
+            return row_insights, global_insights
+
+        df = deepcopy(ns.data)
+        if ns.tss.is_timeseries:
+            df = self._preprocess_ts_df(df, ns)
+
+        df = df[self.input_cols].fillna(0)
+        row_insights['pyod_anomaly'] = pyod_explainer.predict(df).astype(bool)  # binary labels (0: inlier, 1: outlier)
+        return row_insights, global_insights
+
+    def _preprocess_ts_df(self, df: pd.DataFrame, ns: SimpleNamespace) -> pd.DataFrame:
+        for gcol in ns.tss.group_by:
+            df[gcol] = self.ordinal_encoders[gcol].transform(df[gcol].values.reshape(-1, 1).astype(np.object)).flatten()
+
+        for w in range(ns.tss.window + 1):
+            df[f'__pyod_window_{ns.tss.window - w}'] = df[f'__mdb_ts_previous_{self.target}'].apply(lambda x: x[w])
+        return df


Test patch:
diff --git a/tests/unit_tests/analysis/test_pyod.py b/tests/unit_tests/analysis/test_pyod.py
new file mode 100644
--- /dev/null
+++ b/tests/unit_tests/analysis/test_pyod.py
@@ -0,0 +1,46 @@
+import unittest
+import pandas as pd
+from lightwood.analysis import PyOD
+from lightwood.api.high_level import ProblemDefinition, json_ai_from_problem
+from lightwood.api.high_level import code_from_json_ai, predictor_from_code
+
+
+class TestPyOD(unittest.TestCase):
+    def test_0_pyod_analysis(self):
+        if PyOD is None:
+            print('Skipping this test since PyOD library is not installed')
+            return
+
+        dfs = [
+            pd.read_csv('tests/data/concrete_strength.csv'),
+            pd.read_csv('tests/data/hdi.csv'),
+            pd.read_csv('tests/data/house_sales.csv'),
+        ]
+        dicts = [
+            {'target': 'concrete_strength', 'time_aim': 40},
+            {'target': 'Development Index', 'time_aim': 40},
+            {'target': 'MA', 'time_aim': 40, 'timeseries_settings': {
+                'group_by': ['bedrooms', 'type'],
+                'horizon': 4,
+                'order_by': 'saledate',
+                'window': 8
+            }},
+        ]
+
+        for df, d in zip(dfs, dicts):
+            pdef = ProblemDefinition.from_dict(d)
+            json_ai = json_ai_from_problem(df, problem_definition=pdef)
+
+            json_ai.analysis_blocks = [{
+                'module': 'lightwood.analysis.PyOD',
+                'args': {}
+            }]
+
+            code = code_from_json_ai(json_ai)
+            predictor = predictor_from_code(code)
+
+            predictor.learn(df)
+            predictions = predictor.predict(df.head())
+
+            self.assertIn('pyod_explainer', predictor.runtime_analyzer)
+            self.assertIn('pyod_anomaly', predictions.columns)
diff --git a/tests/unit_tests/analysis/test_shap.py b/tests/unit_tests/analysis/test_shap.py
--- a/tests/unit_tests/analysis/test_shap.py
+++ b/tests/unit_tests/analysis/test_shap.py
@@ -5,7 +5,7 @@
 from lightwood.api.high_level import code_from_json_ai, predictor_from_code
 
 
-class TestBasic(unittest.TestCase):
+class TestSHAP(unittest.TestCase):
     def test_0_shap_analysis(self):
         if ShapleyValues is None:
             print('Skipping this test since the Shapley values library is not installed')