Repository: skekre98/simple_learn (language: python)

Issue:
Create SimpleRegressorList class
**Description of feature/enhancement**
Create a class to keep track of the top N best performing regression models on a certain data set.

**Description of implementation**
Users should be able to export a model at a certain index. This list should fine tune each algorithm's hyper-parameters to be the most accurate and keep track of values such as model accuracy and latency.

**Why this is a good feature/enhancement**
This will allow users to pick and choose algorithms based on personal preference. 

**Additional context**
Model attributes should be as descriptive as possible.



Discussion / hints:
Hi @skekre98, I'd like to take a look at this issue and possibly #24 and see what I can do. I've taken a look at the SimpleClassifierList class as well as the discussion in #38 and would like to try implementing this class based on what I've seen if that works for you.
@sh-biswas that would be great! Would you mind waiting till we close #38 to open a PR so we have a finalized version of `SimpleRegressor`? I'll go ahead and assign the issue over to you for track keeping.
Yes, sounds good!
@sh-biswas SimpleRegressor was merged in #38. Please take a look at it when working on this issue :)  
@sh-biswas any progress on this issue?
@skekre98 sorry for my absence, university kept me quite busy this week. I will start this issue tonight and will let you know when I open a PR!
@sh-biswas cool, no rush :) 

Fix patch:
diff --git a/scripts/custom.sh b/scripts/custom.sh
index ef9525f..d72c26a 100755
--- a/scripts/custom.sh
+++ b/scripts/custom.sh
@@ -86,5 +86,6 @@ function run_tests {
     python -W ignore -m unittest tests/classifiers/simple_classifier_tests.py -v
     python -W ignore -m unittest tests/classifiers/simple_classifier_list_tests.py -v
     python -W ignore -m unittest tests/regressors/simple_regressor_tests.py -v
+    python -W ignore -m unittest tests/regressors/simple_regressor_list_tests.py -v
 
 }
diff --git a/simple_learn/regressors/__init__.py b/simple_learn/regressors/__init__.py
index e10dd57..b75900c 100644
--- a/simple_learn/regressors/__init__.py
+++ b/simple_learn/regressors/__init__.py
@@ -19,3 +19,7 @@
 # SOFTWARE.
 
 from simple_learn.regressors.simple_regressor import SimpleRegressor
+from simple_learn.regressors.simple_regressor_list import (
+    SimpleRegressorList,
+    SimpleRegressorListObject,
+)
diff --git a/simple_learn/regressors/simple_regressor_list.py b/simple_learn/regressors/simple_regressor_list.py
new file mode 100644
index 0000000..1612f2c
--- /dev/null
+++ b/simple_learn/regressors/simple_regressor_list.py
@@ -0,0 +1,204 @@
+# Copyright (c) 2020 Sharvil Kekre skekre98
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import json
+import logging
+import time
+
+import numpy as np
+from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
+from sklearn.model_selection import GridSearchCV
+from sklearn.utils import all_estimators
+
+from simple_learn.encoders import simple_model_encoder
+from simple_learn.regressors import SimpleRegressor
+from simple_learn.regressors.param_grid import model_param_map
+
+
+class SimpleRegressorListObject:
+    """
+    A class used to keep track of SimpleRegressors and corresponding rank for
+    terminal display
+
+    '''
+
+    Attributes
+    ----------
+    rgr : simple_learn.regressors.SimpleRegressor
+        the SimpleRegressor object
+    rank : int
+        The associated rank for regressor
+    """
+
+    def __init__(self, rgr, rank):
+        self.rgr = rgr
+        self.rank = rank
+
+    def __str__(self):
+
+        for k in self.rgr.attributes:
+            if type(self.rgr.attributes[k]) == np.int64:
+                self.rgr.attributes[k] = int(self.rgr.attributes[k])
+
+        attr = {
+            "Type": self.rgr.name,
+            "Rank": self.rank,
+            "Training Duration": "{}s".format(self.rgr.train_duration),
+            "GridSearch Duration": "{}s".format(self.rgr.gridsearch_duration),
+            "Parameters": self.rgr.attributes,
+            "Metrics": self.rgr.metrics,
+            "Index": self.rank - 1,
+        }
+
+        str_out = json.dumps(attr, cls=simple_model_encoder.npEncoder, indent=4)
+        return str_out
+
+    def __repr__(self):
+        attr = {
+            "Type": self.rgr.name,
+            "Rank": self.rank,
+            "Training Duration": "{}s".format(self.rgr.train_duration),
+            "GridSearch Duration": "{}s".format(self.rgr.gridsearch_duration),
+            "Parameters": self.rgr.attributes,
+            "Metrics": self.rgr.metrics,
+            "Index": self.rank - 1,
+        }
+
+        repr_out = json.dumps(attr, cls=simple_model_encoder.npEncoder, indent=4)
+        return repr_out
+
+
+class SimpleRegressorList:
+    """
+    A class used to maintain ranked list of SimpleRegressors
+
+    '''
+
+    Attributes
+    ----------
+    ranked_list : list
+        the ranked list of SimpleRegressors
+    metric : str {auto, mae, mse, r2}
+        the scoring metric for ranking models
+    logger : logging.Logger
+        logger for notifying user of warnings
+
+    Methods
+    -------
+    fit(train_x, train_y, folds=3)
+        Fits a given dataset onto SimpleRegressor and creates a ranked list based
+        on scores
+    pop(index = 0)
+        Removes a SimpleRegressor at a specific index for usage
+    """
+
+    def __init__(self, scoring="auto"):
+        self.ranked_list = []
+        metric_map = {
+            "auto": "Training Score",
+            "mae": "Mean Absolute Error",
+            "mse": "Mean Squared Error",
+            "r2": "R-Squared",
+        }
+        self.metric = metric_map[scoring]
+        self.logger = logging.getLogger()
+
+    def __str__(self):
+        r = 1
+        res = []
+        for rgr in self.ranked_list:
+            obj = SimpleRegressorListObject(rgr, r)
+            res.append(str(obj))
+            r += 1
+        return "\n".join(res) if len(res) > 1 else "The List is Empty!"
+
+    def __repr__(self):
+        r = 1
+        res = []
+        for rgr in self.ranked_list:
+            obj = SimpleRegressorListObject(rgr, r)
+            res.append(str(obj))
+            r += 1
+        return "\n".join(res) if len(res) > 1 else "The List is Empty!"
+
+    def fit(self, train_x, train_y, folds=3):
+        """
+        Trains all regressors from parameter grid by running model algorithm search.
+
+        Creates a ranked list of models based on selected scoring metric.
+
+        Parameters
+        ----------
+        train_x : numpy.ndarray
+            The features for training regression model
+        train_y : numpy.ndarray
+            The corresponding label for feature array
+        folds : int, optional
+            The number of folds for cross validation
+        """
+
+        estimators = all_estimators(type_filter="regressor")
+        for name, RegressionClass in estimators:
+            if name in model_param_map:
+                param_grid = model_param_map[name]
+                grid_rgr = GridSearchCV(
+                    RegressionClass(),
+                    param_grid,
+                    cv=folds,
+                    scoring="neg_root_mean_squared_error",
+                    verbose=0,
+                    n_jobs=-1,
+                    error_score="raise",
+                )
+                start = time.time()
+                try:
+                    grid_rgr.fit(train_x, train_y)
+                except BaseException as error:
+                    self.logger.warning(f"{name} failed due to, Error : {error}.")
+                    continue
+                end = time.time()
+                rgr = SimpleRegressor()
+                rgr.metrics["Training Score"] = -grid_rgr.best_score_
+                pred_y = grid_rgr.predict(train_x)
+                rgr.metrics["Mean Absolute Error"] = mean_absolute_error(
+                    train_y, pred_y
+                )
+                rgr.metrics["Mean Squared Error"] = mean_squared_error(train_y, pred_y)
+                rgr.metrics["R-Squared"] = r2_score(train_y, pred_y)
+                rgr.sk_model = grid_rgr.best_estimator_
+                rgr.name = name
+                rgr.attributes = grid_rgr.best_params_
+                rgr.train_duration = grid_rgr.refit_time_
+                rgr.gridsearch_duration = end - start
+                self.ranked_list.append(rgr)
+            metrik = lambda rgr: rgr.metrics[self.metric]
+            self.ranked_list.sort(reverse=False, key=metrik)
+
+    def pop(self, index=0):
+        """Removes SimpleRegressor from a specific index in ranked list.
+
+        Parameters
+        ----------
+        index : int
+            The index corresponding to the SimpleRegressor
+            being removed from ranked list
+        """
+
+        return self.ranked_list.pop(index)


Test patch:
diff --git a/tests/regressors/simple_regressor_list_tests.py b/tests/regressors/simple_regressor_list_tests.py
new file mode 100644
index 0000000..0dd8598
--- /dev/null
+++ b/tests/regressors/simple_regressor_list_tests.py
@@ -0,0 +1,77 @@
+# Copyright (c) 2020 Sharvil Kekre skekre98
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import unittest
+import warnings
+
+warnings.filterwarnings("ignore")
+
+import numpy as np
+from scipy.stats import levene
+from sklearn import datasets
+from sklearn.datasets import make_regression
+
+from simple_learn.regressors import SimpleRegressorList
+
+
+class TestSimpleRegressorList(unittest.TestCase):
+    def test_regression(self):
+        true_x, true_y = make_regression(
+            n_features=4, n_informative=2, random_state=10, shuffle=False
+        )
+
+        rgr_list = SimpleRegressorList()
+        rgr_list.fit(true_x, true_y)
+        self.assertTrue(len(rgr_list.ranked_list) > 0)
+
+        rgr1 = rgr_list.pop(1)
+        rgr0 = rgr_list.pop()
+
+        pred0_y = rgr0.predict(true_x)
+        pred1_y = rgr1.predict(true_x)
+
+        stat0, p0 = levene(true_y, pred0_y)
+        stat1, p1 = levene(true_y, pred1_y)
+
+        self.assertTrue(p0 > 0.05)
+
+    def test_boston(self):
+        boston = datasets.load_boston()
+        true_x = boston.data
+        true_y = boston.target
+
+        rgr_list = SimpleRegressorList()
+        rgr_list.fit(true_x, true_y)
+        self.assertTrue(len(rgr_list.ranked_list) > 0)
+
+        rgr1 = rgr_list.pop(1)
+        rgr0 = rgr_list.pop()
+
+        pred0_y = rgr0.predict(true_x)
+        pred1_y = rgr1.predict(true_x)
+
+        stat0, p0 = levene(true_y, pred0_y)
+        stat1, p1 = levene(true_y, pred1_y)
+
+        self.assertTrue(p0 > 0.05)
+
+
+if __name__ == "__main__":
+    unittest.main()