Repository: mindsdb/lightwood

Issue:
Merge `fixed_confidence` with `anomaly_error_rate` argument
Both are used basically for the same purpose, they should be merged.


Discussion:

Resolving patch:
diff --git a/lightwood/analysis/nc/calibrate.py b/lightwood/analysis/nc/calibrate.py
--- a/lightwood/analysis/nc/calibrate.py
+++ b/lightwood/analysis/nc/calibrate.py
@@ -283,15 +283,10 @@ def explain(self, row_insights: pd.DataFrame, global_insights: Dict[str, object]
 
                 # convert (B, 2, 99) into (B, 2) given width or error rate constraints
                 if is_numerical:
-                    significances = ns.pred_args.fixed_confidence
-                    if significances is not None:
-                        confs = all_confs[:, :, int(100 * (1 - significances)) - 1]
-                    else:
-                        error_rate = ns.pred_args.anomaly_error_rate if is_anomaly_task else None
-                        significances, confs = get_numeric_conf_range(all_confs,
-                                                                      df_target_stddev=ns.analysis['df_target_stddev'],
-                                                                      positive_domain=self.positive_domain,
-                                                                      error_rate=error_rate)
+                    significances, confs = get_numeric_conf_range(all_confs,
+                                                                  df_target_stddev=ns.analysis['df_target_stddev'],
+                                                                  positive_domain=self.positive_domain,
+                                                                  fixed_conf=ns.pred_args.fixed_confidence)
                     result.loc[X.index, 'lower'] = confs[:, 0]
                     result.loc[X.index, 'upper'] = confs[:, 1]
                 else:
@@ -324,17 +319,17 @@ def explain(self, row_insights: pd.DataFrame, global_insights: Dict[str, object]
                                 # predict and get confidence level given width or error rate constraints
                                 if is_numerical:
                                     all_confs = icp.predict(X.values)
-                                    error_rate = ns.pred_args.anomaly_error_rate if is_anomaly_task else None
+                                    fixed_conf = ns.pred_args.fixed_confidence
                                     significances, confs = get_numeric_conf_range(
                                         all_confs,
                                         df_target_stddev=ns.analysis['df_target_stddev'],
                                         positive_domain=self.positive_domain,
                                         group=frozenset(group),
-                                        error_rate=error_rate
+                                        fixed_conf=fixed_conf
                                     )
 
                                     # only replace where grouped ICP is more informative (i.e. tighter)
-                                    if ns.pred_args.fixed_confidence is None:
+                                    if fixed_conf is None:
                                         default_widths = result.loc[X.index, 'upper'] - result.loc[X.index, 'lower']
                                         grouped_widths = np.subtract(confs[:, 1], confs[:, 0])
                                         insert_index = (default_widths > grouped_widths)[lambda x: x.isin([True])].index
diff --git a/lightwood/analysis/nc/util.py b/lightwood/analysis/nc/util.py
--- a/lightwood/analysis/nc/util.py
+++ b/lightwood/analysis/nc/util.py
@@ -100,7 +100,7 @@ def get_numeric_conf_range(
         positive_domain: bool = False,
         std_tol: int = 1,
         group: Optional[str] = '__default',
-        error_rate: float = None
+        fixed_conf: float = None
 ):
     """
     Gets prediction bounds for numerical targets, based on ICP estimation and width tolerance.
@@ -110,14 +110,14 @@ def get_numeric_conf_range(
     :param positive_domain: Flag that indicates whether target is expected to be a positive number.
     :param std_tol: Tolerance for automatic confidence level selection; bigger tolerance means higher confidence, in general.
     :param group: For tasks with multiple different target groups (where each may have a different std_dev), indicates what group is being considered.
-    :param error_rate: Pre-determined error rate for the ICP, 0-1 bounded. Can be specified to bypass automatic confidence/bound detection, or to adjust the threshold sensitivity in anomaly detection tasks.
+    :param fixed_conf: Pre-determined confidence for the ICP, 0-1 bounded. Can be specified to bypass automatic confidence/bound detection, or to adjust the threshold sensitivity in anomaly detection tasks.
     
     :return: array with confidence for each data instance, along with lower and upper bounds for each prediction.
     """  # noqa
-    if not isinstance(error_rate, float):
+    if not isinstance(fixed_conf, float):
         error_rate = None
 
-    if error_rate is None:
+    if fixed_conf is None:
         significances = []
         conf_ranges = []
         std_dev = df_target_stddev[group]
@@ -143,8 +143,8 @@ def get_numeric_conf_range(
         conf_ranges = np.array(conf_ranges)
     else:
         # fixed error rate
-        error_rate = max(0.01, min(1.0, error_rate))
-        conf = 1 - error_rate
+        conf = max(0.01, min(1.0, fixed_conf))
+        error_rate = 1 - conf
         conf_idx = int(100 * error_rate) - 1
         conf_ranges = all_confs[:, :, conf_idx]
         significances = [conf for _ in range(conf_ranges.shape[0])]
diff --git a/lightwood/api/types.py b/lightwood/api/types.py
--- a/lightwood/api/types.py
+++ b/lightwood/api/types.py
@@ -570,10 +570,7 @@ class PredictionArguments:
     instead of returning only the predicted class, the output additionally includes the assigned probability for
     each class.   
     :param all_mixers: forces an ensemble to return predictions emitted by all its internal mixers. 
-    :param fixed_confidence: For analyzer module, specifies a fixed `alpha` confidence for the model calibration so \
-        that predictions, in average, are correct `alpha` percent of the time.
-    :param anomaly_error_rate: Error rate for unsupervised anomaly detection. Bounded between 0.01 and 0.99 \
-        (respectively implies wider and tighter bounds, all other parameters being equal).
+    :param fixed_confidence: Used in the ICP analyzer module, specifies an `alpha` fixed confidence so that predictions, in average, are correct `alpha` percent of the time. For unsupervised anomaly detection, this also translates into the expected error rate. Bounded between 0.01 and 0.99 (respectively implies wider and tighter bounds, all other parameters being equal).
     :param anomaly_cooldown: Sets the minimum amount of timesteps between consecutive firings of the the anomaly \
         detector.
     """  # noqa
@@ -581,7 +578,6 @@ class PredictionArguments:
     predict_proba: bool = True
     all_mixers: bool = False
     fixed_confidence: Union[int, float, None] = None
-    anomaly_error_rate: Union[float, None] = None
     anomaly_cooldown: int = 1
     forecast_offset: int = 0
 
@@ -599,7 +595,6 @@ def from_dict(obj: Dict):
         predict_proba = obj.get('predict_proba', PredictionArguments.predict_proba)
         all_mixers = obj.get('all_mixers', PredictionArguments.all_mixers)
         fixed_confidence = obj.get('fixed_confidence', PredictionArguments.fixed_confidence)
-        anomaly_error_rate = obj.get('anomaly_error_rate', PredictionArguments.anomaly_error_rate)
         anomaly_cooldown = obj.get('anomaly_cooldown', PredictionArguments.anomaly_cooldown)
         forecast_offset = obj.get('forecast_offset', PredictionArguments.forecast_offset)
 
@@ -607,7 +602,6 @@ def from_dict(obj: Dict):
             predict_proba=predict_proba,
             all_mixers=all_mixers,
             fixed_confidence=fixed_confidence,
-            anomaly_error_rate=anomaly_error_rate,
             anomaly_cooldown=anomaly_cooldown,
             forecast_offset=forecast_offset,
         )


Test patch:
diff --git a/tests/integration/advanced/test_timeseries.py b/tests/integration/advanced/test_timeseries.py
--- a/tests/integration/advanced/test_timeseries.py
+++ b/tests/integration/advanced/test_timeseries.py
@@ -105,9 +105,8 @@ def test_0_time_series_grouped_regression(self):
 
         # Check custom ICP params
         test.pop('__mdb_make_predictions')
-        preds = pred.predict(test, {'fixed_confidence': 0.01, 'anomaly_cooldown': 100, 'anomaly_error_rate': 1})
+        preds = pred.predict(test, {'fixed_confidence': 0.01, 'anomaly_cooldown': 100})
         assert all([all([v == 0.01 for v in f]) for f in preds['confidence'].values])
-        assert pred.pred_args.anomaly_error_rate == 1
         assert pred.pred_args.anomaly_cooldown == 100
 
     def test_1_time_series_regression(self):