Repository: mindsdb/lightwood

Issue:
preprocessing of date/datetime are the same in default cleaner
```_standardize_date()``` and ```standardize_datetime()``` are the exact same functions, as seen [here](https://github.com/mindsdb/lightwood/blob/0372d292796a6d1f91ac9df9b8658ad2f128b7c9/lightwood/data/cleaner.py#L26) and [here](https://github.com/mindsdb/lightwood/blob/0372d292796a6d1f91ac9df9b8658ad2f128b7c9/lightwood/data/cleaner.py#L33) respectively




Discussion:

Resolving patch:
diff --git a/lightwood/api/types.py b/lightwood/api/types.py
--- a/lightwood/api/types.py
+++ b/lightwood/api/types.py
@@ -348,7 +348,7 @@ def from_dict(obj: Dict):
         """
         target = obj['target']
         nsubsets = obj.get('nsubsets', 30)
-        pct_invalid = obj.get('pct_invalid', 1)
+        pct_invalid = obj.get('pct_invalid', 2)
         unbias_target = obj.get('unbias_target', True)
         seconds_per_mixer = obj.get('seconds_per_mixer', None)
         seconds_per_encoder = obj.get('seconds_per_encoder', None)
diff --git a/lightwood/data/cleaner.py b/lightwood/data/cleaner.py
--- a/lightwood/data/cleaner.py
+++ b/lightwood/data/cleaner.py
@@ -1,17 +1,130 @@
-from copy import deepcopy
-from lightwood.api.types import TimeseriesSettings
 import re
-from typing import Dict, List
+from copy import deepcopy
+
+import pandas as pd
+import datetime
+from dateutil.parser import parse as parse_dt
+
 from lightwood.api.dtype import dtype
+from lightwood.helpers import text
 from lightwood.helpers.log import log
-from dateutil.parser import parse as parse_dt
-import datetime
-from lightwood.helpers.text import clean_float
-import pandas as pd
+from lightwood.api.types import TimeseriesSettings
 from lightwood.helpers.numeric import can_be_nan_numeric
 
+import numpy as np
+from typing import Dict, List, Optional, Tuple, Callable, Union
+
+
+def cleaner(
+    data: pd.DataFrame,
+    dtype_dict: Dict[str, str],
+    pct_invalid: float,
+    ignore_features: List[str],
+    identifiers: Dict[str, str],
+    target: str,
+    mode: str,
+    timeseries_settings: TimeseriesSettings,
+    anomaly_detection: bool,
+    custom_cleaning_functions: Dict[str, str] = {}
+) -> pd.DataFrame:
+    """
+    The cleaner is a function which takes in the raw data, plus additional information about it's types and about the problem. Based on this it generates a "clean" representation of the data, where each column has an ideal standaridzed type and all malformed or otherwise missing or invalid elements are turned into ``None``
+
+    :param data: The raw data
+    :param dtype_dict: Type information for each column
+    :param pct_invalid: How much of each column can be invalid
+    :param ignore_features: Columns that we want to ignore
+    :param identifiers: A dict containing all identifier typed columns
+    :param target: The target columns
+    :param mode: Can be "predict" or "train"
+    :param timeseries_settings: Timeseries related settings, only relevant for timeseries predictors, otherwise can be the default object
+    :param anomaly_detection: Are we detecting anomalies with this predictor?
+
+    :returns: The cleaned data
+    """ # noqa
+
+    data = _remove_columns(data, ignore_features, identifiers, target, mode, timeseries_settings,
+                           anomaly_detection, dtype_dict)
+
+    for col in _get_columns_to_clean(data, dtype_dict, mode, target):
+        # Get and apply a cleaning function for each data type
+        # If you want to customize the cleaner, it's likely you can to modify ``get_cleaning_func``
+        data[col] = data[col].apply(get_cleaning_func(dtype_dict[col], custom_cleaning_functions)
+                                    ).replace({np.nan: None})
+        # If a column has too many None values, raise an Excpetion
+        # Figure out how to reintroduce later, maybe a custom flag, `crash for too much invalid data`?
+        # _check_if_invalid(data[col], pct_invalid, col)
+
+    return data
+
+
+def _check_if_invalid(new_data: pd.Series, pct_invalid: float, col_name: str) -> bool:
+    """ Checks how many invalid data points there are """
+
+    chk_invalid = (
+        100
+        * (len(new_data) - len([x for x in new_data if x is not None]))
+        / len(new_data)
+    )
+
+    if chk_invalid > pct_invalid:
+        err = f'Too many ({chk_invalid}%) invalid values in column {col_name}nam'
+        log.error(err)
+        raise Exception(err)
+
+
+def get_cleaning_func(data_dtype: dtype, custom_cleaning_functions: Dict[str, str]) -> Callable:
+    """
+    For the provided data type, provide the appropriate cleaning function. Below are the defaults, users can either override this function OR impose a custom block.
+
+    :param data_dtype: The data-type (inferred from a column) as prescribed from ``api.dtype``
+
+    :returns: The appropriate function that will pre-process (clean) data of specified dtype.
+    """ # noqa
+    if data_dtype in custom_cleaning_functions:
+        clean_func = eval(custom_cleaning_functions[data_dtype])
+
+    elif data_dtype in (dtype.date, dtype.datetime):
+        clean_func = _standardize_datetime
+
+    elif data_dtype in (dtype.float, dtype.tsarray):
+        clean_func = _clean_float
+
+    elif data_dtype in (dtype.integer):
+        clean_func = _clean_int
+
+    elif data_dtype in (dtype.array):
+        clean_func = _standardize_array
+
+    elif data_dtype in (dtype.tags):
+        clean_func = _tags_to_tuples
+
+    elif data_dtype in (dtype.quantity):
+        clean_func = _clean_quantity
+
+    elif data_dtype in (
+        dtype.short_text,
+        dtype.rich_text,
+        dtype.categorical,
+        dtype.binary,
+    ):
+        clean_func = _clean_text
+
+    else:
+        raise ValueError(f"{data_dtype} is not supported. Check lightwood.api.dtype")
+
+    return clean_func
+
 
-def _to_datetime(element):
+# ------------------------- #
+# Temporal Cleaning
+# ------------------------- #
+
+
+def _standardize_datetime(element: object) -> Optional[float]:
+    """
+    Parses an expected date-time element. Intakes an element that can in theory be anything.
+    """
     try:
         date = parse_dt(str(element))
     except Exception:
@@ -20,115 +133,150 @@ def _to_datetime(element):
         except Exception:
             return None
 
-    return date
-
-
-def _standardize_date(element):
-    date = _to_datetime(element)
-    if date is None:
-        return None
     return date.timestamp()
 
 
-def _standardize_datetime(element):
-    date = _to_datetime(element)
-    if date is None:
-        return None
-    return date.timestamp()
+# ------------------------- #
+# Tags/Sequences
+# ------------------------- #
 
+# TODO Make it split on something other than commas
+def _tags_to_tuples(tags_str: str) -> Tuple[str]:
+    """
+    Converts comma-separated values into a tuple to preserve a sequence/array.
 
-def _tags_to_tuples(tags_str):
+    Ex:
+    >> x = 'apples, oranges, bananas'
+    >> _tags_to_tuples(x)
+    >> ('apples', 'oranges', 'bananas')
+    """
     try:
-        return tuple([x.strip() for x in tags_str.split(',')])
+        return tuple([x.strip() for x in tags_str.split(",")])
     except Exception:
         return tuple()
 
 
-def _clean_float_or_none(element):
-    try:
-        calened_float = clean_float(element)
-        if can_be_nan_numeric(calened_float):
-            return None
-        return calened_float
-    except Exception:
-        return None
+def _standardize_array(element: object) -> Optional[Union[List[float], float]]:
+    """
+    Given an array of numbers in the form ``[1, 2, 3, 4]``, converts into a numerical sequence.
 
+    :param element: An array-like element in a sequence
+    :returns: standardized array OR scalar number IF edge case
 
-def _standardize_array(element):
+    Ex of edge case:
+    >> element = [1]
+    >> _standardize_array(element)
+    >> 1
+    """
     try:
         element = str(element)
-        element = element.rstrip(']').lstrip('[')
-        element = element.rstrip(' ').lstrip(' ')
-        element = element.replace(', ', ' ').replace(',', ' ')
+        element = element.rstrip("]").lstrip("[")
+        element = element.rstrip(" ").lstrip(" ")
+        element = element.replace(", ", " ").replace(",", " ")
         # Handles cases where arrays are numbers
-        if ' ' not in element:
-            element = _clean_float_or_none(element)
+        if " " not in element:
+            element = _clean_float(element)
         else:
-            element = [float(x) for x in element.split(' ')]
+            element = [float(x) for x in element.split(" ")]
     except Exception:
         pass
 
     return element
 
 
-def _clean_value(element: object, data_dtype: str):
-    if data_dtype in (dtype.date):
-        element = _standardize_date(element)
+# ------------------------- #
+# Integers/Floats/Quantities
+# ------------------------- #
 
-    if data_dtype in (dtype.datetime):
-        element = _standardize_datetime(element)
+def _clean_float(element: object) -> Optional[float]:
+    """
+    Given an element, converts it into a numeric format. If element is NaN, or inf, then returns None.
+    """
+    try:
+        cleaned_float = text.clean_float(element)
+        if can_be_nan_numeric(cleaned_float):
+            return None
+        return cleaned_float
+    except Exception:
+        return None
 
-    if data_dtype in (dtype.float):
-        element = float(_clean_float_or_none(element))
-    if data_dtype in (dtype.integer):
-        element = int(_clean_float_or_none(element))
 
-    if data_dtype in (dtype.array):
-        element = _standardize_array(element)
+def _clean_int(element: object) -> Optional[int]:
+    element = _clean_float(element)
+    if element is not None:
+        element = int(element)
+    return element
 
-    if data_dtype in (dtype.tags):
-        element = _tags_to_tuples(element)
 
-    if data_dtype in (dtype.quantity):
-        element = float(re.sub("[^0-9.,]", '', element).replace(',', '.'))
+def _clean_quantity(element: object) -> Optional[float]:
+    element = float(re.sub("[^0-9.,]", "", str(element)).replace(",", "."))
+    return _clean_float(element)
 
-    if data_dtype in (dtype.short_text, dtype.rich_text, dtype.categorical, dtype.binary):
-        element = str(element)
 
-    return element
+def _clean_text(element: object) -> str:
+    return str(element)
 
 
-def clean_empty_targets(df: pd.DataFrame, target: str) -> pd.DataFrame:
+def _rm_rows_w_empty_targets(df: pd.DataFrame, target: str) -> pd.DataFrame:
+    """
+    Drop any rows that have targets as unknown. Targets are necessary to train.
+
+    :param df: The input dataframe including the target value
+    :param target: the column name that is the output target variable
+
+    :returns: Data with any target smissing
+    """
+    # Compare length before/after
     len_before = len(df)
+
+    # Use Pandas ```dropna``` to omit any rows with missing values for targets; these cannot be trained
     df = df.dropna(subset=[target])
+
+    # Compare length with after
     len_after = len(df)
     nr_removed = len_before - len_after
+
     if nr_removed != 0:
         log.warning(
-            f'Removed {nr_removed} rows due to the target value missing. Training with rows without a target value makes no sense, please avoid this!') # noqa
+            f"Removed {nr_removed} rows because target was missing. Training on these rows is not possible."
+        )  # noqa
 
     return df
 
 
-def cleaner(
-        data: pd.DataFrame, dtype_dict: Dict[str, str],
-        pct_invalid: float, ignore_features: List[str],
-        identifiers: Dict[str, str],
-        target: str, mode: str, timeseries_settings: TimeseriesSettings, anomaly_detection: bool) -> pd.DataFrame:
-    # Drop columns we don't want to use
+def _remove_columns(data: pd.DataFrame, ignore_features: List[str], identifiers: Dict[str, object], target: str,
+                    mode: str, timeseries_settings: TimeseriesSettings, anomaly_detection: bool,
+                    dtype_dict: Dict[str, dtype]) -> pd.DataFrame:
+    """
+    Drop columns we don't want to use in order to train or predict
+
+    :param data: The raw data
+    :param dtype_dict: Type information for each column
+    :param ignore_features: Columns that we want to ignore
+    :param identifiers: A dict containing all identifier typed columns
+    :param target: The target columns
+    :param mode: Can be "predict" or "train"
+    :param timeseries_settings: Timeseries related settings, only relevant for timeseries predictors, otherwise can be the default object
+    :param anomaly_detection: Are we detecting anomalies with this predictor?
+
+    :returns: A (new) dataframe without the dropped columns
+    """ # noqa
     data = deepcopy(data)
-    to_drop = [*ignore_features, [x for x in identifiers.keys() if x != target]]
-    exceptions = ['__mdb_make_predictions']
+    to_drop = [*ignore_features, *[x for x in identifiers.keys() if x != target],
+               *[x for x in data.columns if x in dtype_dict and dtype_dict[x] == dtype.invalid]]
+    exceptions = ["__mdb_make_predictions"]
     for col in to_drop:
-        try:
-            data = data.drop(columns=[col])
-        except Exception:
-            pass
-
-    if mode == 'train':
-        data = clean_empty_targets(data, target)
-    if mode == 'predict':
-        if target in data.columns and not timeseries_settings.use_previous_target and not anomaly_detection:
+        if col in data.columns:
+            data.drop(columns=[col], inplace=True)
+
+    if mode == "train":
+        data = _rm_rows_w_empty_targets(data, target)
+    if mode == "predict":
+        if (
+            target in data.columns
+            and not timeseries_settings.use_previous_target
+            and not anomaly_detection
+        ):
             data = data.drop(columns=[target])
 
     # Drop extra columns
@@ -136,33 +284,24 @@ def cleaner(
         if name not in dtype_dict and name not in exceptions:
             data = data.drop(columns=[name])
 
-    # Standardize content
-    for name, data_dtype in dtype_dict.items():
-        if mode == 'predict':
+    return data
+
+
+def _get_columns_to_clean(data: pd.DataFrame, dtype_dict: Dict[str, dtype], mode: str, target: str) -> List[str]:
+    """
+    :param data: The raw data
+    :param dtype_dict: Type information for each column
+    :param target: The target columns
+    :param mode: Can be "predict" or "train"
+
+    :returns: A list of columns that we want to clean
+    """ # noqa
+
+    cleanable_columns = []
+    for name, _ in dtype_dict.items():
+        if mode == "predict":
             if name == target:
                 continue
-        if name in to_drop:
-            continue
-        if name not in data.columns:
-            if '__mdb_ts_previous' not in name:
-                data[name] = [None] * len(data)
-            continue
-
-        new_data = []
-        for element in data[name]:
-            try:
-                new_data.append(_clean_value(element, data_dtype))
-            except Exception as e:
-                new_data.append(None)
-                log.warning(
-                    f'Unable to parse elemnt: {element} or type {data_dtype} from column {name}. Excetpion: {e}')
-
-        pct_invalid = 100 * (len(new_data) - len([x for x in new_data if x is not None])) / len(new_data)
-
-        if pct_invalid > pct_invalid:
-            err = f'Too many ({pct_invalid}%) invalid values in column {name} of type {data_dtype}'
-            log.error(err)
-            raise Exception(err)
-
-        data[name] = new_data
-    return data
+        if name in data.columns:
+            cleanable_columns.append(name)
+    return cleanable_columns
diff --git a/lightwood/data/statistical_analysis.py b/lightwood/data/statistical_analysis.py
--- a/lightwood/data/statistical_analysis.py
+++ b/lightwood/data/statistical_analysis.py
@@ -10,16 +10,16 @@
 from lightwood.helpers.log import log
 from lightwood.api.dtype import dtype
 from scipy.stats import entropy
-from lightwood.data.cleaner import _clean_float_or_none
+from lightwood.data.cleaner import _clean_float
 
 
 def get_datetime_histogram(data: pd.Series, bins: int) -> Dict[str, list]:
     """Generates the histogram for date and datetime types
     """
     if isinstance(data[0], float) or isinstance(data[0], int):
-        data = [_clean_float_or_none(x) for x in data]
+        data = [_clean_float(x) for x in data]
     else:
-        data = [_clean_float_or_none(parse_dt(str(x)).timestamp()) for x in data]
+        data = [_clean_float(parse_dt(str(x)).timestamp()) for x in data]
 
     Y, X = np.histogram(data, bins=min(bins, len(set(data))),
                         range=(min(data), max(data)), density=False)
@@ -44,7 +44,7 @@ def get_numeric_histogram(data: pd.Series, data_dtype: dtype, bins: int) -> Dict
             new_data.extend(arr)
         data = new_data
 
-    data = [_clean_float_or_none(x) for x in data]
+    data = [_clean_float(x) for x in data]
 
     Y, X = np.histogram(data, bins=min(bins, len(set(data))),
                         range=(min(data), max(data)), density=False)


Test patch:
diff --git a/tests/integration/basic/test_weird_target_dist.py b/tests/integration/basic/test_weird_target_dist.py
--- a/tests/integration/basic/test_weird_target_dist.py
+++ b/tests/integration/basic/test_weird_target_dist.py
@@ -1,6 +1,7 @@
 import unittest
 import pandas as pd
 from lightwood.api.types import ProblemDefinition
+from lightwood import dtype
 
 
 class TestBasic(unittest.TestCase):
@@ -28,4 +29,5 @@ def test_0_unkown_cateogires_in_test(self):
         predictor = predictor_from_problem(df, ProblemDefinition.from_dict(
                                            {'target': target, 'time_aim': 60, 'unbias_target': True}))
         predictor.learn(df)
+        assert predictor.model_analysis.dtypes['target'] == dtype.categorical
         predictor.predict(df)