Repository: mindsdb/lightwood

Issue:
Generate histogram for all types
Title says it all, for types where generating one is pointless just have an empty one, I guess, because otherwise the GUI can't display it...


Discussion:

Resolving patch:
diff --git a/lightwood/data/cleaner.py b/lightwood/data/cleaner.py
--- a/lightwood/data/cleaner.py
+++ b/lightwood/data/cleaner.py
@@ -60,11 +60,13 @@ def _standardize_array(element):
         element = element.rstrip(']').lstrip('[')
         element = element.rstrip(' ').lstrip(' ')
         element = element.replace(', ', ' ').replace(',', ' ')
-        # Weird edge case in which arrays are actually numbers -_-
+        # Handles cases where arrays are numbers
         if ' ' not in element:
-            return _clean_float_or_none(element)
+            element = _clean_float_or_none(element)
     except Exception:
-        return element
+        pass
+
+    return element
 
 
 def _clean_value(element: object, data_dtype: str):
diff --git a/lightwood/data/statistical_analysis.py b/lightwood/data/statistical_analysis.py
--- a/lightwood/data/statistical_analysis.py
+++ b/lightwood/data/statistical_analysis.py
@@ -1,9 +1,10 @@
 from typing import Dict
 import pandas as pd
 import numpy as np
-
+import datetime
+from dateutil.parser import parse as parse_dt
 from lightwood.api import StatisticalAnalysis, ProblemDefinition
-from lightwood.helpers.numeric import filter_nan
+from lightwood.helpers.numeric import filter_nan_and_none
 from lightwood.helpers.seed import seed
 from lightwood.data.cleaner import cleaner
 from lightwood.helpers.log import log
@@ -12,10 +13,40 @@
 from lightwood.data.cleaner import _clean_float_or_none
 
 
-def get_numeric_histogram(data, data_dtype):
+def get_datetime_histogram(data: pd.Series, bins: int) -> Dict[str, list]:
+    """Generates the histogram for date and datetime types
+    """
+    if isinstance(data[0], float) or isinstance(data[0], int):
+        data = [_clean_float_or_none(x) for x in data]
+    else:
+        data = [_clean_float_or_none(parse_dt(str(x)).timestamp()) for x in data]
+
+    Y, X = np.histogram(data, bins=min(bins, len(set(data))),
+                        range=(min(data), max(data)), density=False)
+
+    X = X[:-1].tolist()
+    Y = Y.tolist()
+
+    X = [str(datetime.datetime.fromtimestamp(x)) for x in X]
+    return {
+        'x': X,
+        'y': Y
+    }
+
+
+def get_numeric_histogram(data: pd.Series, data_dtype: dtype, bins: int) -> Dict[str, list]:
+    """Generate the histogram for integer and float typed data
+    """
+    # Handle arrays that are actual arrays and not things that become arrays later
+    if ' ' in str(data[0]):
+        new_data = []
+        for list_str in data:
+            new_data.extend([float(x) for x in list_str.split(' ')])
+        data = new_data
+
     data = [_clean_float_or_none(x) for x in data]
 
-    Y, X = np.histogram(data, bins=min(50, len(set(data))),
+    Y, X = np.histogram(data, bins=min(bins, len(set(data))),
                         range=(min(data), max(data)), density=False)
     if data_dtype == dtype.integer:
         Y, X = np.histogram(data, bins=[int(round(x)) for x in X], density=False)
@@ -84,18 +115,20 @@ def statistical_analysis(data: pd.DataFrame,
     for col in df.columns:
         histograms[col] = None
         buckets[col] = None
-        if dtypes[col] in (dtype.categorical, dtype.binary, dtype.date):
-            hist = dict(df[col].value_counts().apply(lambda x: x / len(df[col])))
+        if dtypes[col] in (dtype.categorical, dtype.binary, dtype.tags):
+            hist = dict(df[col].value_counts())
             histograms[col] = {
                 'x': list([str(x) for x in hist.keys()]),
                 'y': list(hist.values())
             }
             buckets[col] = histograms[col]['x']
         elif dtypes[col] in (dtype.integer, dtype.float, dtype.array):
-            histograms[col] = get_numeric_histogram(filter_nan(df[col]), dtypes[col])
+            histograms[col] = get_numeric_histogram(filter_nan_and_none(df[col]), dtypes[col], 50)
             buckets[col] = histograms[col]['x']
+        elif dtypes[col] in (dtype.date, dtype.datetime):
+            histograms[col] = get_datetime_histogram(filter_nan_and_none(df[col]), 50)
         else:
-            histograms[col] = {'x': [], 'y': []}
+            histograms[col] = {'x': ['Unknown'], 'y': [len(df[col])]}
             buckets[col] = []
 
     # get observed classes, used in analysis
diff --git a/lightwood/helpers/numeric.py b/lightwood/helpers/numeric.py
--- a/lightwood/helpers/numeric.py
+++ b/lightwood/helpers/numeric.py
@@ -1,15 +1,14 @@
-'''
-To: future intern reading this bit of code thinking "this is dumb, we could just check for isnan and isinf
+from typing import Iterable
 
-No, no you can't, this will break subtly in a bunch of edge cases.
 
-See the example (+ comments) in this article: https://cerebralab.com/Exceptions_as_control_flow
+def can_be_nan_numeric(value: object) -> bool:
+    """
+    Tells us if **value** might be nan or inf or some other numeric value\
+        (i.e. which can be cast as `float`) that is not actually a number.\
+        Name is vague because I'm not 100% sure of all the edge cases of numeric\
+        values that have number-like type behavior.
+    """
 
-Sorry, python is dumb, but the spirit of your investigation wasn't, try again
-'''
-
-
-def can_be_nan_numeric(value):
     try:
         value = str(value)
         value = float(value)
@@ -25,5 +24,5 @@ def can_be_nan_numeric(value):
     return isnan
 
 
-def filter_nan(series):
-    return [x for x in series if not can_be_nan_numeric(x)]
+def filter_nan_and_none(series: Iterable) -> list:
+    return [x for x in series if not can_be_nan_numeric(x) and x is not None]


Test patch:
diff --git a/tests/unit_tests/data/test_infer_types.py b/tests/integration/basic/test_analyze_dataset.py
similarity index 92%
rename from tests/unit_tests/data/test_infer_types.py
rename to tests/integration/basic/test_analyze_dataset.py
--- a/tests/unit_tests/data/test_infer_types.py
+++ b/tests/integration/basic/test_analyze_dataset.py
@@ -1,5 +1,5 @@
 from mindsdb_datasources import FileDS
-from lightwood import infer_types
+from lightwood import analyze_dataset
 from lightwood.api import dtype
 import unittest
 from itertools import cycle
@@ -16,11 +16,11 @@
 
 
 class TestInferTypes(unittest.TestCase):
-    def test_infer_types_on_home_rentlas(self):
+    def test_analyze_home_rentlas(self):
         datasource = FileDS(
             "https://raw.githubusercontent.com/mindsdb/mindsdb-examples/master/classics/home_rentals/dataset/train.csv"
         )
-        type_information = infer_types(datasource.df, pct_invalid=0)
+        type_information = analyze_dataset(datasource.df).type_information
 
         self.assertTrue(
             type_information.dtypes["number_of_rooms"] == dtype.categorical)
@@ -47,8 +47,9 @@ def test_infer_types_on_home_rentlas(self):
         for k in type_information.identifiers:
             self.assertTrue(type_information.identifiers[k] is None)
 
-    def test_type_deduction(self):
-        n_points = 100
+    def test_with_generated(self):
+        # Must be an even number
+        n_points = 134
 
         # Apparently for n_category_values = 10 it doesn't work
         n_category_values = 4
@@ -90,7 +91,8 @@ def test_type_deduction(self):
             }
         )
 
-        type_information = infer_types(df, pct_invalid=0)
+        analysis = analyze_dataset(df)
+        type_information = analysis.type_information
         for col_name in df.columns:
             expected_type = test_column_types[col_name]
             print(
@@ -98,6 +100,18 @@ def test_type_deduction(self):
             )
             assert type_information.dtypes[col_name] == expected_type
 
+        stats = analysis.statistical_analysis
+        for k in stats.histograms:
+            if k != 'sequential_array':
+                assert np.sum(stats.histograms[k]['y']) == n_points
+
+        assert set(stats.histograms['short_text']['x']) == set(['Unknown'])
+        assert set(stats.histograms['rich_text']['x']) == set(['Unknown'])
+        assert set(stats.histograms['rich_text']['x']) == set(['Unknown'])
+        # 50 is a magic number, when we change this, tests must change
+        assert len(set(stats.histograms['date_timestamp']['x'])) == 50
+        assert len(set(stats.histograms['date_date']['x'])) == 50
+
 
 '''
     def test_deduce_foreign_key(self):