Repository: mindsdb/mindsdb
Pull request: Data Analytics Backend V2
* Resolves #450 
* Resolves #435 (Not fully, but what it doesn't, is now part of other issues)
* Adds a new data cleaning phase that runs multiple times (much less complex than other phases).
* Major refactoring to the StatsGenerator that greatly simplify the logic and remove a bunch of bugs, that being said, the original `stats` object returned by it should remain unchanged, all the new stuff is inside `stats_v2`, which will soon (TM) fully replace it.

Problem statement:
Various improvements to the Data Analysis
There will be various improvements to the Data Analysis phase for version 1.20.0 (compatible with scout version 2.0).

For now I will keep track of all of them here, if some of them require their own issue we can open them later.

Currently still discussing the changes around this internally, two big things would be:

* Re-work some of the scores
* Add predictive power based correlation score. i.e. how correlated/similar two columns are based on whether or not they can predict each other. could also be expanded further into creating some sort of causal graph between certain columns or column combinations.

Numerical Int histogram - Int only buckets
For a data subtype of integer, our histogram x-axis and buckets should begin/end at integer values.

Hints from issue discussion:

Commit message: removed debugging logic

Patch to review:
diff --git a/mindsdb/__about__.py b/mindsdb/__about__.py
index 064f9484ebf..5628949b489 100755
--- a/mindsdb/__about__.py
+++ b/mindsdb/__about__.py
@@ -1,6 +1,6 @@
 __title__ = 'MindsDB'
 __package_name__ = 'mindsdb'
-__version__ = '1.17.9'
+__version__ = '1.18.0'
 __description__ = "MindsDB's goal is to make it very simple for developers to use the power of artificial neural networks in their projects."
 __email__ = "jorge@mindsdb.com"
 __author__ = 'MindsDB Inc'
diff --git a/mindsdb/libs/controllers/predictor.py b/mindsdb/libs/controllers/predictor.py
index 23d4ee808af..c8aa24e3fcd 100755
--- a/mindsdb/libs/controllers/predictor.py
+++ b/mindsdb/libs/controllers/predictor.py
@@ -215,6 +215,9 @@ def get_model_data(self, model_name=None, lmd=None):
         # ADAPTOR CODE
         amd = {}
 
+        if 'stats_v2' in lmd:
+            amd['data_analysis_v2'] = lmd['stats_v2']
+
         if lmd['current_phase'] == MODEL_STATUS_TRAINED:
             amd['status'] = 'complete'
         elif lmd['current_phase'] == MODEL_STATUS_ERROR:
diff --git a/mindsdb/libs/controllers/transaction.py b/mindsdb/libs/controllers/transaction.py
index cd345ebf85d..2c36f339eea 100755
--- a/mindsdb/libs/controllers/transaction.py
+++ b/mindsdb/libs/controllers/transaction.py
@@ -140,8 +140,11 @@ def _execute_analyze(self):
         self._call_phase_module(module_name='DataExtractor')
         self.save_metadata()
 
+        self._call_phase_module(module_name='DataCleaner', stage=0)
+        self.save_metadata()
+
         self.lmd['current_phase'] = MODEL_STATUS_DATA_ANALYSIS
-        self._call_phase_module(module_name='StatsGenerator', input_data=self.input_data, modify_light_metadata=True, hmd=self.hmd)
+        self._call_phase_module(module_name='StatsGenerator', input_data=self.input_data, hmd=self.hmd)
         self.save_metadata()
 
         self.lmd['current_phase'] = MODEL_STATUS_DONE
@@ -159,14 +162,16 @@ def _execute_learn(self):
             self._call_phase_module(module_name='DataExtractor')
             self.save_metadata()
 
+            self._call_phase_module(module_name='DataCleaner', stage=0)
+            self.save_metadata()
+
             self.lmd['current_phase'] = MODEL_STATUS_DATA_ANALYSIS
-            if 'skip_stats_generation' in self.lmd and self.lmd['skip_stats_generation'] == True:
-                self.load_metadata()
-            else:
-                self.save_metadata()
-                self._call_phase_module(module_name='StatsGenerator', input_data=self.input_data, modify_light_metadata=True, hmd=self.hmd)
-                self.save_metadata()
+            self._call_phase_module(module_name='StatsGenerator', input_data=self.input_data, hmd=self.hmd)
+            self.save_metadata()
 
+            self._call_phase_module(module_name='DataCleaner', stage=0)
+            self.save_metadata()
+            
             self._call_phase_module(module_name='DataSplitter')
 
             self._call_phase_module(module_name='DataTransformer', input_data=self.input_data)
diff --git a/mindsdb/libs/phases/data_cleaner/__init__.py b/mindsdb/libs/phases/data_cleaner/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/mindsdb/libs/phases/data_cleaner/data_cleaner.py b/mindsdb/libs/phases/data_cleaner/data_cleaner.py
new file mode 100644
index 00000000000..60f6ab3c3ce
--- /dev/null
+++ b/mindsdb/libs/phases/data_cleaner/data_cleaner.py
@@ -0,0 +1,27 @@
+from mindsdb.libs.phases.base_module import BaseModule
+from mindsdb.libs.data_types.mindsdb_logger import log
+
+
+class DataCleaner(BaseModule):
+    def _cleanup_w_missing_targets(self, df):
+        initial_len = len(df)
+        df = df.dropna(subset=self.transaction.lmd['predict_columns'])
+        no_dropped = len(df) - initial_len
+        if no_dropped > 0:
+            self.log.warning(f'Dropped {no_dropped} rows because they had null values in one or more of the columns that we are trying to predict. Please always provide non-null values in the columns you want to predict !')
+        return df
+
+    def _cleanup_ignored(self, df):
+        for col_name in df.columns.values:
+            if len(df[col_name].dropna()) < 1:
+                self.transaction.lmd['columns_to_ignore'].append(col_name)
+                self.transaction.lmd['empty_columns'].append(col_name)
+                self.log.warning(f'Column "{col_name}" is empty ! We\'ll go ahead and ignore it, please make sure you gave mindsdb the correct data.')
+
+        df = df.drop(columns=self.transaction.lmd['columns_to_ignore'])
+        return df
+
+    def run(self, stage):
+        if stage == 0:
+            self.transaction.input_data.data_frame = self._cleanup_w_missing_targets(self.transaction.input_data.data_frame)
+            self.transaction.input_data.data_frame = self._cleanup_ignored(self.transaction.input_data.data_frame)
diff --git a/mindsdb/libs/phases/data_splitter/data_splitter.py b/mindsdb/libs/phases/data_splitter/data_splitter.py
index 4136f947a14..5df9e81a8c2 100644
--- a/mindsdb/libs/phases/data_splitter/data_splitter.py
+++ b/mindsdb/libs/phases/data_splitter/data_splitter.py
@@ -5,17 +5,7 @@
 
 
 class DataSplitter(BaseModule):
-    def _cleanup_w_missing_targets(self, df):
-        initial_len = len(df)
-        df = df.dropna(subset=self.transaction.lmd['predict_columns'])
-        no_dropped = len(df) - initial_len
-        if no_dropped > 0:
-            self.log.warning(f'Dropped {no_dropped} rows because they had null values in one or more of the columns that we are trying to predict. Please always provide non-null values in the columns you want to predict !')
-        return df
-
     def run(self):
-        self.transaction.input_data.data_frame = self._cleanup_w_missing_targets(self.transaction.input_data.data_frame)
-
         group_by = self.transaction.lmd['model_group_by']
         if group_by is None or len(group_by) == 0:
             group_by = []
diff --git a/mindsdb/libs/phases/stats_generator/data_preparation.py b/mindsdb/libs/phases/stats_generator/data_preparation.py
new file mode 100644
index 00000000000..759c6aaf459
--- /dev/null
+++ b/mindsdb/libs/phases/stats_generator/data_preparation.py
@@ -0,0 +1,36 @@
+import random
+import numpy as np
+
+from mindsdb.external_libs.stats import calculate_sample_size
+from mindsdb.libs.constants.mindsdb import *
+from mindsdb.libs.helpers.text_helpers import clean_float
+
+def sample_data(df, sample_margin_of_error, sample_confidence_level, log):
+    population_size = len(df)
+
+    sample_size = int(calculate_sample_size(population_size, sample_margin_of_error, sample_confidence_level)) if population_size > 50 else population_size
+    sample_size_pct = sample_size*100/population_size
+
+    # get the indexes of randomly selected rows given the population size
+    input_data_sample_indexes = random.sample(range(population_size), sample_size)
+
+    log.info(f'Analyzing a sample of {sample_size} from a total population of {population_size}, this is equivalent to {sample_size_pct}% of your data.')
+
+    return df.iloc[input_data_sample_indexes]
+
+
+def clean_int_and_date_data(col_data, log):
+    cleaned_data = []
+
+    for ele in col_data:
+        if str(ele) not in ['', str(None), str(False), str(np.nan), 'NaN', 'nan', 'NA', 'null'] and ele != '' and ele != '\r' and ele != '\n':
+            try:
+                cleaned_data.append(clean_float(ele))
+            except Exception as e1:
+                try:
+                    cleaned_data.append(parse_datetime(str(ele)).timestamp())
+                except Exception as e2:
+                    log.warning(f'Failed to parser numerical value with error chain:\n {e1} -> {e2}\n')
+                    cleaned_data.append(0)
+                    
+    return cleaned_data
diff --git a/mindsdb/libs/phases/stats_generator/scores.py b/mindsdb/libs/phases/stats_generator/scores.py
index d9dd453756f..9a11761ef29 100644
--- a/mindsdb/libs/phases/stats_generator/scores.py
+++ b/mindsdb/libs/phases/stats_generator/scores.py
@@ -169,12 +169,13 @@ def compute_lof_score(stats, columns, col_name):
     np_col_data = np.array(columns[col_name]).reshape(-1, 1)
     lof = LocalOutlierFactor(contamination='auto')
     outlier_scores = lof.fit_predict(np_col_data)
-    outlier_indexes = [i for i in range(len(columns[col_name])) if outlier_scores[i] < -0.8]
+
+    outliers = [columns[col_name][i] for i in range(len(columns[col_name])) if outlier_scores[i] < -0.8]
 
     return {
-        'lof_outliers': outlier_indexes
-        ,'lof_based_outlier_score': round(10 * (1 - len(outlier_indexes)/len(columns[col_name])))
-        ,'percentage_of_log_based_outliers': (len(outlier_indexes)/len(columns[col_name])) * 100
+        'lof_outliers': outliers
+        ,'lof_based_outlier_score': round(10 * (1 - len(outliers)/len(columns[col_name])))
+        ,'percentage_of_log_based_outliers': (len(outliers)/len(columns[col_name])) * 100
         ,'lof_based_outlier_score_description':"""
         The higher this score, the more outliers your dataset has. This is based on distance from the center of 20 clusters as constructed via KNN.
         """
diff --git a/mindsdb/libs/phases/stats_generator/stats_generator.py b/mindsdb/libs/phases/stats_generator/stats_generator.py
index 81506e8ee26..05ee106016e 100755
--- a/mindsdb/libs/phases/stats_generator/stats_generator.py
+++ b/mindsdb/libs/phases/stats_generator/stats_generator.py
@@ -5,10 +5,9 @@
 import sndhdr
 import logging
 from collections import Counter
-#import multiprocessing
 
 import numpy as np
-import scipy.stats as st
+from scipy.stats import entropy
 from dateutil.parser import parse as parse_datetime
 from sklearn.metrics.pairwise import cosine_similarity
 from sklearn.cluster import MiniBatchKMeans
@@ -18,11 +17,10 @@
 from mindsdb.config import CONFIG
 from mindsdb.libs.constants.mindsdb import *
 from mindsdb.libs.phases.base_module import BaseModule
-from mindsdb.libs.helpers.text_helpers import splitRecursive, clean_float, cast_string_to_python_type
+from mindsdb.libs.helpers.text_helpers import splitRecursive, clean_float
 from mindsdb.libs.helpers.debugging import *
-from mindsdb.external_libs.stats import calculate_sample_size
 from mindsdb.libs.phases.stats_generator.scores import *
-
+from mindsdb.libs.phases.stats_generator.data_preparation import sample_data, clean_int_and_date_data
 
 class StatsGenerator(BaseModule):
     """
@@ -271,25 +269,6 @@ def _get_column_data_type(self, data, data_frame, col_name):
 
         return curr_data_type, curr_data_subtype, type_dist, subtype_dist, additional_info, 'Column ok'
 
-    @staticmethod
-    def clean_int_and_date_data(col_data):
-        cleaned_data = []
-
-        for value in col_data:
-            if value != '' and value != '\r' and value != '\n':
-                cleaned_data.append(value)
-
-        cleaned_data_new = []
-
-        for ele in cleaned_data:
-            if str(ele) not in ['', str(None), str(False), str(np.nan), 'NaN', 'nan', 'NA', 'null']:
-                try:
-                    cleaned_data_new.append(clean_float(ele))
-                except:
-                    cleaned_data_new.append(parse_datetime(str(ele)).timestamp())
-
-        return cleaned_data_new
-
     @staticmethod
     def get_words_histogram(data, is_full_text=False):
         """ Returns an array of all the words that appear in the dataset and the number of times each word appears in the dataset """
@@ -318,22 +297,23 @@ def get_histogram(data, data_type=None, data_subtype=None, full_text=None, hmd=N
             is_full_text = True if data_subtype == DATA_SUBTYPES.TEXT else False
             return StatsGenerator.get_words_histogram(data, is_full_text), None
         elif data_type == DATA_TYPES.NUMERIC or data_subtype == DATA_SUBTYPES.TIMESTAMP:
-            data = StatsGenerator.clean_int_and_date_data(data)
-            y, x = np.histogram(data, bins=50, range=(min(data),max(data)), density=False)
-            x = x[:-1]
+            Y, X = np.histogram(data, bins=min(50,len(set(data))), range=(min(data),max(data)), density=False)
+            if data_subtype == DATA_SUBTYPES.INT:
+                Y, X = np.histogram(data, bins=[int(round(x)) for x in X], density=False)
+
+            X = X[:-1]
             #x = (x + np.roll(x, -1))[:-1] / 2.0 <--- original code, was causing weird bucket values when we had outliers
-            x = x.tolist()
-            y = y.tolist()
+
             return {
-                'x': x
-                ,'y': y
-            }, None
+                'x': X.tolist()
+                ,'y': Y.tolist()
+            }, X
         elif data_type == DATA_TYPES.CATEGORICAL or data_subtype == DATA_SUBTYPES.DATE :
             histogram = Counter(data)
             return {
                 'x': list(map(str,histogram.keys())),
                 'y': list(histogram.values())
-            }, None
+            }, list(histogram.values())
         elif data_subtype == DATA_SUBTYPES.IMAGE:
             image_hashes = []
             for img_path in data:
@@ -524,200 +504,87 @@ def _log_interesting_stats(self, stats):
                 # Functionality is specific to mindsdb logger
                 pass
 
-    def run(self, input_data, modify_light_metadata, hmd=None, print_logs=True):
+    def run(self, input_data, hmd=None, print_logs=True):
         """
         # Runs the stats generation phase
         # This shouldn't alter the columns themselves, but rather provide the `stats` metadata object and update the types for each column
         # A lot of information about the data distribution and quality will  also be logged to the server in this phase
         """
 
-        ''' @TODO Uncomment when we need multiprocessing, possibly disable on OSX
-        no_processes = multiprocessing.cpu_count() - 2
-        if no_processes < 1:
-            no_processes = 1
-        pool = multiprocessing.Pool(processes=no_processes)
-        '''
+        stats = {}
+        stats_v2 = {}
+        col_data_dict = {}
+
         if print_logs == False:
             self.log = logging.getLogger('null-logger')
             self.log.propagate = False
 
-        # we dont need to generate statistic over all of the data, so we subsample, based on our accepted margin of error
-        population_size = len(input_data.data_frame)
+        sample_df = sample_data(input_data.data_frame, self.transaction.lmd['sample_margin_of_error'], self.transaction.lmd['sample_confidence_level'], self.log)
 
-        if population_size < 50:
-            sample_size = population_size
-        else:
-            sample_size = int(calculate_sample_size(population_size=population_size, margin_error=self.transaction.lmd['sample_margin_of_error'], confidence_level=self.transaction.lmd['sample_confidence_level']))
-            #if sample_size > 3000 and sample_size > population_size/8:
-            #    sample_size = min(round(population_size/8),3000)
+        for col_name in self.transaction.lmd['empty_columns']:
+            stats_v2[col_name]['empty'] = {'is_empty': True}
 
-        # get the indexes of randomly selected rows given the population size
-        input_data_sample_indexes = random.sample(range(population_size), sample_size)
-        self.log.info('population_size={population_size},  sample_size={sample_size}  {percent:.2f}%'.format(population_size=population_size, sample_size=sample_size, percent=(sample_size/population_size)*100))
+        for col_name in sample_df.columns.values:
+            stats_v2[col_name] = {}
+            stats[col_name] = {}
 
-        all_sampled_data = input_data.data_frame.iloc[input_data_sample_indexes]
+            len_wo_nulls = len(input_data.data_frame[col_name].dropna())
+            len_w_nulls = len(input_data.data_frame[col_name])
+            len_unique = len(set(input_data.data_frame[col_name]))
+            stats_v2[col_name]['empty'] = {
+                'empty_cells': len_w_nulls - len_wo_nulls
+                ,'empty_percentage': 100 * round((len_w_nulls - len_wo_nulls)/len_w_nulls,3)
+                ,'is_empty': False
+            }
 
-        stats = {}
-        col_data_dict = {}
+            col_data = sample_df[col_name].dropna()
 
-        for col_name in all_sampled_data.columns.values:
-            if col_name in self.transaction.lmd['columns_to_ignore']:
-                continue
+            data_type, data_subtype, data_type_dist, data_subtype_dist, additional_info, column_status = self._get_column_data_type(col_data, input_data.data_frame, col_name)
 
-            col_data = all_sampled_data[col_name].dropna()
-            full_col_data = all_sampled_data[col_name]
+            stats_v2[col_name]['typing'] = {
+                'data_type': data_type
+                ,'data_subtype': data_subtype
+                ,'data_type_dist': data_type_dist
+                ,'data_subtype_dist': data_subtype_dist
+            }
 
-            data_type, curr_data_subtype, data_type_dist, data_subtype_dist, additional_info, column_status = self._get_column_data_type(col_data, input_data.data_frame, col_name)
+            for k  in stats_v2[col_name]['typing']: stats[col_name][k] = stats_v2[col_name]['typing'][k]
 
-            if column_status == 'Column empty':
-                if modify_light_metadata:
-                    self.transaction.lmd['empty_columns'].append(col_name)
-                    logging.warning(f'The "{col_name}" column is empty, it will be ignored, please make sure the data in the column is correct !')
-                    self.transaction.lmd['columns_to_ignore'].append(col_name)
-                continue
+            # Do some temporary processing for timestamp and numerical values
+            if data_type == DATA_TYPES.NUMERIC or data_subtype == DATA_SUBTYPES.TIMESTAMP:
+                col_data = clean_int_and_date_data(col_data, self.log)
 
-            new_col_data = []
-
-            if curr_data_subtype == DATA_SUBTYPES.TIMESTAMP: #data_type == DATA_TYPES.DATE:
-                for element in col_data:
-                    if str(element) in [str(''), str(None), str(False), str(np.nan), 'NaN', 'nan', 'NA', 'null']:
-                        new_col_data.append(None)
-                    else:
-                        try:
-                            new_col_data.append(int(parse_datetime(element).timestamp()))
-                        except:
-                            self.log.warning(f'Could not convert string from col "{col_name}" to date and it was expected, instead got: {element}')
-                            new_col_data.append(None)
-                col_data = new_col_data
-            if data_type == DATA_TYPES.NUMERIC or curr_data_subtype == DATA_SUBTYPES.TIMESTAMP:
-                histogram, _ = StatsGenerator.get_histogram(col_data, data_type=data_type, data_subtype=curr_data_subtype)
-                x = histogram['x']
-                y = histogram['y']
-
-                col_data = StatsGenerator.clean_int_and_date_data(col_data)
-                # This means the column is all nulls, which we don't handle at the moment
-                if len(col_data) < 1:
-                    return None
-
-                if len(col_data) > 0:
-                    max_value = max(col_data)
-                    min_value = min(col_data)
-                    mean = np.mean(col_data)
-                    median = np.median(col_data)
-                    var = np.var(col_data)
-                    skew = st.skew(col_data)
-                    kurtosis = st.kurtosis(col_data)
-                else:
-                    max_value = 0
-                    min_value = 0
-                    mean = 0
-                    median = 0
-                    var = 0
-                    skew = 0
-                    kurtosis = 0
-
-                is_float = True if max([1 if int(i) != i else 0 for i in col_data]) == 1 else False
-
-                col_stats = {
-                    'data_type': data_type,
-                    'data_subtype': curr_data_subtype,
-                    "mean": mean,
-                    "median": median,
-                    "variance": var,
-                    "skewness": skew,
-                    "kurtosis": kurtosis,
-                    "max": max_value,
-                    "min": min_value,
-                    "is_float": is_float,
-                    "histogram": {
-                        "x": x,
-                        "y": y
-                    },
-                    "percentage_buckets": histogram['x']#xp
+            hist_data = col_data
+            if data_type == DATA_TYPES.CATEGORICAL:
+                hist_data = input_data.data_frame[col_name]
+                stats_v2[col_name]['unique'] = {
+                    'unique_values': len_unique
+                    ,'unique_percentage': 100 * round((len_w_nulls - len_unique)/len_w_nulls,8)
                 }
 
-            elif data_type == DATA_TYPES.CATEGORICAL or curr_data_subtype == DATA_SUBTYPES.DATE:
-                histogram, _ = StatsGenerator.get_histogram(input_data.data_frame[col_name], data_type=data_type, data_subtype=curr_data_subtype)
+            histogram, percentage_buckets = StatsGenerator.get_histogram(hist_data, data_type=data_type, data_subtype=data_subtype)
 
-                col_stats = {
-                    'data_type': data_type,
-                    'data_subtype': curr_data_subtype,
-                    "histogram": histogram,
-                    "percentage_buckets": histogram['x']
-                }
+            stats[col_name]['histogram'] = histogram
+            stats[col_name]['percentage_buckets'] = percentage_buckets
+            stats_v2[col_name]['histogram'] = histogram
+            stats_v2[col_name]['percentage_buckets'] = percentage_buckets
 
-            elif curr_data_subtype == DATA_SUBTYPES.IMAGE:
-                histogram, percentage_buckets = StatsGenerator.get_histogram(col_data, data_subtype=curr_data_subtype)
+            stats[col_name]['empty_cells'] = stats_v2[col_name]['empty']['empty_cells']
+            stats[col_name]['empty_percentage'] = stats_v2[col_name]['empty']['empty_percentage']
 
-                col_stats = {
-                    'data_type': data_type,
-                    'data_subtype': curr_data_subtype,
-                    'percentage_buckets': percentage_buckets,
-                    'histogram': histogram
-                }
-
-            elif curr_data_subtype == DATA_SUBTYPES.ARRAY:
-                col_stats = {
-                    'data_type': data_type,
-                    'data_subtype': curr_data_subtype,
-                    'percentage_buckets': None,
-                    'histogram': None
-                }
-
-            # @TODO This is probably wrong, look into it a bit later
-            else:
-                # see if its a sentence or a word
-                histogram, _ = StatsGenerator.get_histogram(col_data, data_type=data_type, data_subtype=curr_data_subtype)
-                dictionary = list(histogram.keys())
-
-                # if no words, then no dictionary
-                if len(col_data) == 0:
-                    dictionary_available = False
-                    dictionary_lenght_percentage = 0
-                    dictionary = []
-                else:
-                    dictionary_available = True
-                    dictionary_lenght_percentage = len(
-                        dictionary) / len(col_data) * 100
-                    # if the number of uniques is too large then treat is a text
-                    is_full_text = True if curr_data_subtype == DATA_SUBTYPES.TEXT else False
-                    if dictionary_lenght_percentage > 10 and len(col_data) > 50 and is_full_text==False:
-                        dictionary = []
-                        dictionary_available = False
-
-                col_stats = {
-                    'data_type': data_type,
-                    'data_subtype': curr_data_subtype,
-                    "dictionary": dictionary,
-                    "dictionaryAvailable": dictionary_available,
-                    "dictionaryLenghtPercentage": dictionary_lenght_percentage,
-                    "histogram": histogram
-                }
-            stats[col_name] = col_stats
-            stats[col_name]['data_type_dist'] = data_type_dist
-            stats[col_name]['data_subtype_dist'] = data_subtype_dist
-            stats[col_name]['column'] = col_name
-
-            empty_count = len(full_col_data) - len(col_data)
-
-            stats[col_name]['empty_cells'] = empty_count
-            stats[col_name]['empty_percentage'] = empty_count * 100 / len(full_col_data)
+            stats_v2[col_name]['additional_info'] = additional_info
             for k in additional_info:
                 stats[col_name][k] = additional_info[k]
 
             col_data_dict[col_name] = col_data
 
-        for col_name in all_sampled_data.columns:
-            if col_name in self.transaction.lmd['columns_to_ignore']:
-                continue
+        for col_name in sample_df.columns:
+            data_type = stats_v2[col_name]['typing']['data_type']
+            data_subtype = stats_v2[col_name]['typing']['data_subtype']
 
-            # Use the multiprocessing pool for computing scores which take a very long time to compute
             # For now there's only one and computing it takes way too long, so this is not enabled
             scores = []
 
-            '''
-            scores.append(pool.apply_async(compute_clf_based_correlation_score, args=(stats, all_sampled_data, col_name)))
-            '''
             for score_promise in scores:
                 # Wait for function on process to finish running
                 score = score_promise.get()
@@ -730,7 +597,7 @@ def run(self, input_data, modify_light_metadata, hmd=None, print_logs=True):
                     if 'compute_z_score' in str(score_func) or 'compute_lof_score' in str(score_func):
                         stats[col_name].update(score_func(stats, col_data_dict, col_name))
                     else:
-                        stats[col_name].update(score_func(stats, all_sampled_data, col_name))
+                        stats[col_name].update(score_func(stats, sample_df, col_name))
                 except Exception as e:
                     self.log.warning(e)
 
@@ -747,20 +614,34 @@ def run(self, input_data, modify_light_metadata, hmd=None, print_logs=True):
             if stats[col_name]['is_foreign_key'] and self.transaction.lmd['handle_foreign_keys']:
                 self.transaction.lmd['columns_to_ignore'].append(col_name)
 
-        total_rows = len(input_data.data_frame)
+            # New logic
+            col_data = sample_df[col_name]
 
-        if modify_light_metadata:
-            self.transaction.lmd['column_stats'] = stats
+            if data_type in (DATA_TYPES.NUMERIC,DATA_TYPES.DATE,DATA_TYPES.CATEGORICAL) or data_subtype in (DATA_SUBTYPES.IMAGE):
+                nr_values = sum(stats_v2[col_name]['histogram']['y'])
+                S = entropy([x/nr_values for x in stats_v2[col_name]['histogram']['y']],base=max(2,len(stats_v2[col_name]['histogram']['y'])))
+                stats_v2[col_name]['bias'] = {
+                    'entropy': S
+                }
+                if S < 0.25:
+                    pick_nr = -max(1, int(len(stats_v2[col_name]['histogram']['y'])/10))
+                    stats_v2[col_name]['bias']['biased_buckets'] = [stats_v2[col_name]['histogram']['x'][i] for i in np.array(stats_v2[col_name]['histogram']['y']).argsort()[pick_nr:]]
+
+            if 'lof_outliers' in stats[col_name]:
+                if data_subtype in (DATA_SUBTYPES.INT):
+                    stats[col_name]['lof_outliers'] = [int(x) for x in stats[col_name]['lof_outliers']]
+
+                stats_v2[col_name]['outliers'] = {
+                    'outlier_values': stats[col_name]['lof_outliers']
+                    ,'outlier_score': stats[col_name]['lof_based_outlier_score']
+                }
 
-            self.transaction.lmd['data_preparation']['accepted_margin_of_error'] = self.transaction.lmd['sample_margin_of_error']
+        self.transaction.lmd['column_stats'] = stats
+        self.transaction.lmd['stats_v2'] = stats_v2
 
-            self.transaction.lmd['data_preparation']['total_row_count'] = total_rows
-            self.transaction.lmd['data_preparation']['used_row_count'] = sample_size
+        self.transaction.lmd['data_preparation']['accepted_margin_of_error'] = self.transaction.lmd['sample_margin_of_error']
 
-        ''' @TODO Uncomment when we need multiprocessing, possibly disable on OSX
-        pool.close()
-        pool.join()
-        '''
+        self.transaction.lmd['data_preparation']['total_row_count'] = len(input_data.data_frame)
+        self.transaction.lmd['data_preparation']['used_row_count'] = len(sample_df)
 
         self._log_interesting_stats(stats)
-        return stats
diff --git a/tests/ci_tests/tests.py b/tests/ci_tests/tests.py
index 356a86dcfdc..185b937e8e7 100755
--- a/tests/ci_tests/tests.py
+++ b/tests/ci_tests/tests.py
@@ -20,9 +20,8 @@ def test_model_analysis(amd, to_predict):
     for column, importance in zip(input_importance["x"], input_importance["y"]):
         assert isinstance(column, str)
         assert (len(column) > 0)
-
         assert isinstance(importance, (float, int))
-        assert (importance > 0)
+        assert (importance >= 0 and importance <= 10)
 
 def test_force_vectors(amd, to_predict):
     force_vectors = amd['force_vectors']
@@ -65,7 +64,7 @@ def basic_test(backend='lightwood',use_gpu=True, run_extra=False, IS_CI_TEST=Fal
                     ctn = True
             if ctn:
                 continue
-            
+
             code = os.system(f'python3 ../functional_testing/{py_file}')
             if code != 0:
                 raise Exception(f'Test failed with status code: {code} !')


Code review comments:
diff_hunk: @@ -0,0 +1,36 @@
+import random
+import numpy as np
+
+from mindsdb.external_libs.stats import calculate_sample_size
+from mindsdb.libs.constants.mindsdb import *
+from mindsdb.libs.helpers.text_helpers import clean_float
+
+def sample_data(df, sample_margin_of_error, sample_confidence_level, log):
+    population_size = len(df)
+
+    sample_size = int(calculate_sample_size(population_size, sample_margin_of_error, sample_confidence_level)) if population_size > 50 else population_size
+    sample_size_pct = sample_size*100/population_size
+
+    # get the indexes of randomly selected rows given the population size
+    input_data_sample_indexes = random.sample(range(population_size), sample_size)
+
+    log.info(f'Analyzing a sample of {sample_size} from a total population of {population_size}, this is equivalent to {sample_size_pct}% of your data.')
+
+    return df.iloc[input_data_sample_indexes]
+
+
+def clean_int_and_date_data(col_data, log):
+    cleaned_data = []
+
+    for ele in col_data:
+        if str(ele) not in ['', str(None), str(False), str(np.nan), 'NaN', 'nan', 'NA', 'null'] and ele != '' and ele != '\r' and ele != '\n':
path: mindsdb/libs/phases/stats_generator/data_preparation.py
text: @user1:
We can use `and (not ele or not ele.isspace())` that will check for space, tab, newline, carriage return and feed'\f'