Repository: mindsdb/mindsdb

Issue:
Categorical value get interpreted as texts
Some of the categorical values from the kaggel suicide dataset get interpreted as `text` rather than `categorical`. This might have to do with the stats generator sample size being too small.

See #182 in order to see how this can be replicated.



Discussion:

Resolving patch:
diff --git a/mindsdb/libs/constants/mindsdb.py b/mindsdb/libs/constants/mindsdb.py
--- a/mindsdb/libs/constants/mindsdb.py
+++ b/mindsdb/libs/constants/mindsdb.py
@@ -26,7 +26,7 @@ class DATA_SUBTYPES:
 
     # CATEGORICAL
     SINGLE = 'Binary Category'
-    MULTIPLE = 'Category' # Kind of unclear on the implementation
+    MULTIPLE = 'Category'
 
     # FILE_PATH
     IMAGE = 'Image'
diff --git a/mindsdb/libs/controllers/predictor.py b/mindsdb/libs/controllers/predictor.py
--- a/mindsdb/libs/controllers/predictor.py
+++ b/mindsdb/libs/controllers/predictor.py
@@ -308,9 +308,6 @@ def get_model_data(self, model_name):
             else:
                 icm['importance_score'] = lmd['column_importances'][col]
                 amd['data_analysis']['input_columns_metadata'].append(icm)
-
-
-
         # ADAPTOR CODE
 
         return amd
@@ -507,7 +504,6 @@ def learn(self, to_predict, from_data = None, test_from_data=None, group_by = No
 
             for k in ['from_data', 'test_from_data']:
                 if old_hmd[k] is not None: heavy_transaction_metadata[k] = old_hmd[k]
-
         Transaction(session=self, light_transaction_metadata=light_transaction_metadata, heavy_transaction_metadata=heavy_transaction_metadata, logger=self.log, breakpoint=breakpoint)
 
 
diff --git a/mindsdb/libs/phases/stats_generator/stats_generator.py b/mindsdb/libs/phases/stats_generator/stats_generator.py
--- a/mindsdb/libs/phases/stats_generator/stats_generator.py
+++ b/mindsdb/libs/phases/stats_generator/stats_generator.py
@@ -112,11 +112,12 @@ def _get_text_type(self, data):
             if max_number_of_words < words:
                 max_number_of_words += words
 
-        if max_number_of_words == 1:
-            return DATA_TYPES.CATEGORICAL, DATA_SUBTYPES.SINGLE
-        if max_number_of_words <= 3 and len(key_count) < total_length * 0.8:
-            # @TODO This used to be multiple... but, makes no sense for cateogry, should be discussed
-            return DATA_TYPES.CATEGORICAL, DATA_SUBTYPES.SINGLE
+        # If all sentences are less than or equal and 3 words, assume it's a category rather than a sentence
+        if max_number_of_words <= 3:
+            if len(key_count.keys()) < 3:
+                return DATA_TYPES.CATEGORICAL, DATA_SUBTYPES.SINGLE
+            else:
+                return DATA_TYPES.CATEGORICAL, DATA_SUBTYPES.MULTIPLE
         else:
             return DATA_TYPES.SEQUENTIAL, DATA_SUBTYPES.TEXT
 
@@ -226,24 +227,28 @@ def _get_column_data_type(self, data, col_index, data_array, col_name):
             type_dist[curr_data_type] = type_dist.pop('Unknown')
             subtype_dist[curr_data_subtype] = subtype_dist.pop('Unknown')
 
-        all_values = []
-        for row in data_array:
-            all_values.append(row[col_index])
-
-        all_distinct_vals = set(all_values)
-
-        # Let's chose so random number
-        if (len(all_distinct_vals) < len(all_values)/200) or ( (len(all_distinct_vals) < 120) and (len(all_distinct_vals) < len(all_values)/6) ):
-            curr_data_type = DATA_TYPES.CATEGORICAL
-            if len(all_distinct_vals) < 3:
-                curr_data_subtype = DATA_SUBTYPES.SINGLE
-            else:
-                curr_data_subtype = DATA_SUBTYPES.MULTIPLE
-            type_dist = {}
-            subtype_dist = {}
+        # @TODO: Extremely slow for large datasets, make it faster
+        if curr_data_type != DATA_TYPES.CATEGORICAL:
+            all_values = []
+            for row in data_array:
+                all_values.append(row[col_index])
+
+            all_distinct_vals = set(all_values)
+
+            # The numbers here are picked randomly, the gist of it is that if values repeat themselves a lot we should consider the column to be categorical
+            nr_vals = len(all_values)
+            nr_distinct_vals = len(all_distinct_vals)
+            if nr_vals/15 > nr_distinct_vals:
+                curr_data_type = DATA_TYPES.CATEGORICAL
+                if len(all_distinct_vals) < 3:
+                    curr_data_subtype = DATA_SUBTYPES.SINGLE
+                else:
+                    curr_data_subtype = DATA_SUBTYPES.MULTIPLE
+                type_dist = {}
+                subtype_dist = {}
 
-            type_dist[curr_data_type] = len(data)
-            subtype_dist[curr_data_subtype] = len(data)
+                type_dist[curr_data_type] = len(data)
+                subtype_dist[curr_data_subtype] = len(data)
 
         return curr_data_type, curr_data_subtype, type_dist, subtype_dist, additional_info, 'Column ok'
 


Test patch:
diff --git a/integration_testing/run_travis_tests.py b/integration_testing/run_travis_tests.py
--- a/integration_testing/run_travis_tests.py
+++ b/integration_testing/run_travis_tests.py
@@ -111,7 +111,7 @@ def run_tests():
 
         # Print statements are in for debugging, remove later, but keep the funcion calls to make sure the interface is working
         models = mdb.get_models()
-        amd = mdb.get_model_data(models[0]['name'])
+        amd = mdb.get_model_data('test_one_label_prediction')
         print(amd)
 
     except: