Repository: mindsdb/lightwood

Issue:
`Test` method for PredictorInterface
We should add a new method to the predictor interface for arbitrary metric computation given a dataframe with values for the target column.

```python
pdef = ProblemDefinition.from_dict({...})
json_ai = json_ai_from_problem(train_df, problem_definition=pdef)
predictor_class_code = code_from_json_ai(json_ai)
predictor = predictor_from_code(predictor_class_code)
predictor.learn(train_df)

# existing behavior
predictions = predictor.predict(test_df)

# new behavior

# 1) string for already supported metrics and user-defined modules
metrics = predictor.test(test_df, metrics=['mean_absolute_error', 'my_custom_module.method_name'])

# 2) support for callables and lambdas
from sklearn.metrics import mean_absolute_error
metrics = predictor.test(test_df, metrics=[mean_absolute_error, lambda x, y: abs(x - y)])
```

Returned object should be a Pandas dataframe with the value of each passed metric:

```python
In[1]: metrics
Out[1]:
                             metric_name  value
mean_absolute_error  mean_absolute_error  32433
lambda_0                        lambda_0  45576
```

It may also be worth defining an extra kwarg so that the user can retrieve predictions (as if they were using `predict()`):

```python
metrics, predictions = predictor.test(test_df, metrics=['mean_absolute_error'], return_preds=True)
```


Discussion:

Resolving patch:
diff --git a/lightwood/api/predictor.py b/lightwood/api/predictor.py
--- a/lightwood/api/predictor.py
+++ b/lightwood/api/predictor.py
@@ -28,6 +28,7 @@ class PredictorInterface:
     You can also use the predictor to now estimate new data:
 
     - ``predict``: Deploys the chosen best model, and evaluates the given data to provide target estimates.
+    - ``test``: Similar to predict, but user also passes an accuracy function that will be used to compute a metric with the generated predictions.
     - ``save``: Saves the Predictor object for further use.
 
     The ``PredictorInterface`` is created via J{ai}son's custom code creation. A problem inherits from this class with pre-populated routines to fill out expected results, given the nature of each problem type.
@@ -127,12 +128,27 @@ def adjust(self, new_data: pd.DataFrame, old_data: Optional[pd.DataFrame] = None
 
     def predict(self, data: pd.DataFrame, args: Dict[str, object] = {}) -> pd.DataFrame:
         """
-        Intakes raw data to provide predicted values for your trained model.
+        Intakes raw data to provide model predictions.
+
+        :param data: Data (n_samples, n_columns) that the model will use as input to predict the corresponding target value for each sample.
+        :param args: any parameters used to customize inference behavior. Wrapped as a ``PredictionArguments`` object.
+
+        :returns: A dataframe containing predictions and additional sample-wise information. `n_samples` rows.
+        """  # noqa
+        pass
+
+    def test(
+            self, data: pd.DataFrame, metrics: list, args: Dict[str, object] = {}, strict: bool = False
+    ) -> pd.DataFrame:
+        """
+        Intakes raw data to compute values for a list of provided metrics using a Lightwood predictor.
 
         :param data: Data (n_samples, n_columns) that the model(s) will evaluate on and provide the target prediction.
+        :param metrics: A list of metrics to evaluate the model's performance on.
         :param args: parameters needed to update the predictor ``PredictionArguments`` object, which holds any parameters relevant for prediction.
+        :param strict: If True, the function will raise an error if the model does not support any of the requested metrics. Otherwise it skips them.
 
-        :returns: A dataframe of predictions of the same length of input.
+        :returns: A dataframe with `n_metrics` columns, each cell containing the respective score of each metric.
         """  # noqa
         pass
 
diff --git a/lightwood/helpers/codegen.py b/lightwood/helpers/codegen.py
--- a/lightwood/helpers/codegen.py
+++ b/lightwood/helpers/codegen.py
@@ -505,6 +505,50 @@ def _timed_call(encoded_ds):
 
     predict_body = align(predict_body, 2)
 
+    # ----------------- #
+    # Test Body
+    # ----------------- #
+    test_body = """
+preds = self.predict(data, args)
+preds = preds.rename(columns={'prediction': self.target})
+filtered = []
+
+# filter metrics if not supported
+for metric in metrics:
+    # metric should be one of: an actual function, registered in the model class, or supported by the evaluator
+    if not (callable(metric) or metric in self.accuracy_functions or metric in mdb_eval_accuracy_metrics):
+        if strict:
+            raise Exception(f'Invalid metric: {metric}')
+        else:
+            log.warning(f'Invalid metric: {metric}. Skipping...')
+    else:
+        filtered.append(metric)
+
+metrics = filtered
+try:
+    labels = self.model_analysis.histograms[self.target]['x']
+except:
+    if strict:
+        raise Exception('Label histogram not found')
+    else:
+        label_map = None  # some accuracy functions will crash without this, be mindful
+scores = evaluate_accuracies(
+                data,
+                preds[self.target],
+                self.target,
+                metrics,
+                ts_analysis=self.ts_analysis,
+                labels=labels
+            )
+
+# TODO: remove once mdb_eval returns an actual list
+scores = {k: [v] for k, v in scores.items() if not isinstance(v, list)}
+
+return pd.DataFrame.from_records(scores)  # TODO: add logic to disaggregate per-mixer
+"""
+
+    test_body = align(test_body, 2)
+
     predictor_code = f"""
 {IMPORTS}
 {IMPORT_EXTERNAL_DIRS}
@@ -597,6 +641,11 @@ def adjust(self, train_data: Union[EncodedDs, ConcatedEncodedDs, pd.DataFrame],
     @timed_predictor
     def predict(self, data: pd.DataFrame, args: Dict = {{}}) -> pd.DataFrame:
 {predict_body}
+
+    def test(
+        self, data: pd.DataFrame, metrics: list, args: Dict[str, object] = {{}}, strict: bool = False
+        ) -> pd.DataFrame:
+{test_body}
 """
 
     try:
diff --git a/lightwood/helpers/constants.py b/lightwood/helpers/constants.py
--- a/lightwood/helpers/constants.py
+++ b/lightwood/helpers/constants.py
@@ -45,6 +45,9 @@
 from dataprep_ml.splitters import splitter
 from dataprep_ml.imputers import *
 
+from mindsdb_evaluator import evaluate_accuracies
+from mindsdb_evaluator.accuracy import __all__ as mdb_eval_accuracy_metrics
+
 import pandas as pd
 from typing import Dict, List, Union, Optional
 import os


Test patch:
diff --git a/tests/integration/basic/test_categorical.py b/tests/integration/basic/test_categorical.py
--- a/tests/integration/basic/test_categorical.py
+++ b/tests/integration/basic/test_categorical.py
@@ -69,3 +69,11 @@ def test_2_binary_no_analysis(self):
         self.assertTrue(balanced_accuracy_score(test['target'], predictions['prediction']) > 0.5)
         self.assertTrue('confidence' not in predictions.columns)
 
+        metrics = ['balanced_accuracy_score', 'accuracy_score', 'precision_score']
+        results = predictor.test(test, metrics)
+
+        for metric in metrics:
+            assert metric in results.columns
+            assert 0.5 < results[metric].iloc[0] < 1.0
+
+        print(results)