Repository: mindsdb/mindsdb

Issue:
Document the stats generator's metrics
Document the metrics that are generated in the stats generator page (some docs exist in the comments and in a private google doc, improve on those and move them in an .md format under the `docs/website/docs` dir.


Discussion:

Resolving patch:
diff --git a/mindsdb/libs/controllers/predictor.py b/mindsdb/libs/controllers/predictor.py
--- a/mindsdb/libs/controllers/predictor.py
+++ b/mindsdb/libs/controllers/predictor.py
@@ -134,18 +134,21 @@ def _adapt_column(self, col_stats, col):
                 metrics.append({
                       "type": "score",
                       "score": col_stats['data_type_distribution_score'],
-                      "description": "Scores have no descriptions yet"
+                      "description": col_stats['data_type_distribution_score_description'],
+                      "warning": col_stats['data_type_distribution_score_warning']
                 })
                 metrics.append({
                       "type": "score",
                       "score": col_stats['empty_cells_score'],
-                      "description": "Scores have no descriptions yet"
+                      "description": col_stats['empty_cells_score_description'],
+                      "warning": col_stats['empty_cells_score_warning']
                 })
                 if 'duplicates_score' in col_stats:
                     metrics.append({
                           "type": "score",
                           "score": col_stats['duplicates_score'],
-                          "description": "Scores have no descriptions yet"
+                          "description": col_stats['duplicates_score_description'],
+                          "warning": col_stats['duplicates_score_warning']
                     })
 
             if score == 'variability_score':
@@ -153,37 +156,43 @@ def _adapt_column(self, col_stats, col):
                     metrics.append({
                           "type": "score",
                           "score": col_stats['lof_based_outlier_score'],
-                          "description": "Scores have no descriptions yet"
+                          "description": col_stats['lof_based_outlier_score_description'],
+                          "warning": col_stats['lof_based_outlier_score_warning']
                     })
                     metrics.append({
                           "type": "score",
                           "score": col_stats['z_test_based_outlier_score'],
-                          "description": "Scores have no descriptions yet"
+                          "description": col_stats['z_test_based_outlier_score_description'],
+                          "warning": col_stats['z_test_based_outlier_score_warning']
                     })
                     metrics.append({
                           "type": "score",
                           "score": col_stats['value_distribution_score'],
-                          "description": "Scores have no descriptions yet"
+                          "description": col_stats['value_distribution_score_description'],
+                          "warning": col_stats['value_distribution_score_warning']
                     })
                 else:
                     metrics.append({
                           "type": "score",
                           "score": col_stats['value_distribution_score'],
-                          "description": "Scores have no descriptions yet"
+                          "description": col_stats['value_distribution_score_description'],
+                          "warning": col_stats['value_distribution_score_warning']
                     })
 
             if score == 'redundancy_score':
                 metrics.append({
                       "type": "score",
                       "score": col_stats['similarity_score'],
-                      "description": "Scores have no descriptions yet"
+                      "description": col_stats['similarity_score_description'],
+                      "warning": col_stats['similarity_score_warning']
                 })
 
 
             icm[score.replace('','_score')] = {
                 'score': col_stats[score],
-                'metrics': metrics
-                ,"description": "Scores have no descriptions yet"
+                'metrics': metrics,
+                "description": col_stats[f'{score}_description'],
+                "warning": col_stats[f'{score}_warning']
             }
 
             return icm
diff --git a/mindsdb/libs/phases/stats_generator/stats_generator.py b/mindsdb/libs/phases/stats_generator/stats_generator.py
--- a/mindsdb/libs/phases/stats_generator/stats_generator.py
+++ b/mindsdb/libs/phases/stats_generator/stats_generator.py
@@ -291,6 +291,9 @@ def _compute_value_distribution_score(self, stats, columns, col_name):
             'bucket_probabilities': bucket_probabilities
             ,'value_distribution_score': value_distribution_score
             ,'max_probability_key': max_probability_key
+            ,'value_distribution_score_description': """
+            This score can indicate either biasing towards one specific value in the column or a large number of outliers. So it is a reliable quality indicator but we can't know for which of the two reasons.
+            """
         }
 
         return data
@@ -317,6 +320,9 @@ def _compute_duplicates_score(self, stats, columns, col_name):
         data = {
             'nr_duplicates': nr_duplicates
             ,'duplicates_percentage': nr_duplicates*100/len(columns[col_name])
+            ,'duplicates_score_description':"""
+            The duplicates score consists in the % of duplicate values / 100. So, it can range from 0 (no duplicates) to 1 (all the values have one or more duplicates). This score being large, on it's own, is not necessarily an indicator that your data is of poor quality.
+            """
         }
 
         if stats[col_name]['data_type'] != DATA_TYPES.CATEGORICAL and stats[col_name]['data_type'] != DATA_TYPES.DATE:
@@ -337,7 +343,8 @@ def _compute_empty_cells_score(self, stats, columns, col_name):
             empty_cells_score: A quality score based on the nr of empty cells, ranges from 1 to 0, where 1 is lowest quality and 0 is highest quality.
         """
 
-        return {'empty_cells_score': stats[col_name]['empty_percentage']/100}
+        return {'empty_cells_score': stats[col_name]['empty_percentage']/100
+                ,'empty_cells_score_description':"""This score is computed as the % of empty values / 100. Empty values in a column are always bad for training correctly on that data."""}
 
     def _compute_data_type_dist_score(self, stats, columns, col_name):
         """
@@ -356,7 +363,10 @@ def _compute_data_type_dist_score(self, stats, columns, col_name):
         principal = max(vals)
         total = len(columns[col_name])
         data_type_dist_score = (total - principal)/total
-        return {'data_type_distribution_score': data_type_dist_score}
+        return {'data_type_distribution_score': data_type_dist_score
+        ,'data_type_distribution_score_description':"""
+        This score indicates the amount of data that are not of the same data type as the most commonly detected data type in this column. Note, the most commonly occuring data type is not necessarily the type mindsdb will use to label the column when learning or predicting.
+        """}
 
     def _compute_z_score(self, stats, columns, col_name):
         """
@@ -382,6 +392,9 @@ def _compute_z_score(self, stats, columns, col_name):
             'z_score_outliers': z_score_outlier_indexes
             ,'mean_z_score': np.mean(z_scores)
             ,'z_test_based_outlier_score': len(z_score_outlier_indexes)/len(columns[col_name])
+            ,'z_test_based_outlier_score_description':"""
+            This score indicates the amount of data that are 3 STDs or more away from the mean. That is to say, the amount of data that we consider to be an outlir. A hgih z socre means your data contains a large amount of outliers.
+            """
         }
         return data
 
@@ -412,6 +425,9 @@ def _compute_lof_score(self, stats, columns, col_name):
             'lof_outliers': outlier_indexes
             ,'lof_based_outlier_score': len(outlier_indexes)/len(columns[col_name])
             ,'percentage_of_log_based_outliers': (len(outlier_indexes)/len(columns[col_name])) * 100
+            ,'lof_based_outlier_score_description':"""
+            The higher this score, the more outliers your dataset has. This is based on distance from the center of 20 clusters as constructed via KNN.
+            """
         }
 
 
@@ -442,6 +458,10 @@ def _compute_similariy_score(self, stats, columns, col_name):
             'similarities': similarities
             ,'similarity_score': max_similarity
             ,'most_similar_column_name': list(filter(lambda x: x[1] == max_similarity, similarities))[0][0]
+            ,'similarity_score_description':"""
+            This score is simply a matthews correlation applied between this column and all other column.
+            The score * 100 is the number of values which are similar in the column that is most similar to the scored column.
+            """
         }
 
 
@@ -495,6 +515,9 @@ def _compute_clf_based_correlation_score(self, stats, columns, col_name):
             'correlation_score': prediction_score * highest_correlated_column
             ,'highest_correlation': max(corr_scores)
             ,'most_correlated_column': other_feature_names[corr_scores.index(max(corr_scores))]
+            ,'similarity_score_description':"""
+            A high value for this score means that two of your columns are highly similar. This is done by trying to predict one column using the other via a simple DT.
+            """
         }
 
     def _compute_consistency_score(self, stats, col_name):
@@ -512,7 +535,11 @@ def _compute_consistency_score(self, stats, col_name):
             consistency_score = (col_stats['data_type_distribution_score'] + col_stats['empty_cells_score'])/2.5 + col_stats['duplicates_score']/5
         else:
             consistency_score = (col_stats['data_type_distribution_score'] + col_stats['empty_cells_score'])/2
-        return {'consistency_score': consistency_score}
+        return {'consistency_score': consistency_score
+        ,'consistency_score_description':"""
+        A high value for this score indicates that the data in a column is not very consistent, it's either missing a lot of valus or the type of values it has varries quite a lot (e.g. combination of strings, dates, integers and floats).
+        The data consistency score is mainly based upon the Data Type Distribution Score and the Empty Cells Score, the Duplicates Score is also taken into account if present but with a smaller (2x smaller) bias.
+        """}
 
     def _compute_redundancy_score(self, stats, col_name):
         """
@@ -526,7 +553,11 @@ def _compute_redundancy_score(self, stats, col_name):
         """
         col_stats = stats[col_name]
         redundancy_score = (col_stats['similarity_score'])/1
-        return {'redundancy_score': redundancy_score}
+        return {'redundancy_score': redundancy_score
+            ,'redundancy_score_description':"""
+            A high value in this score indicates the data in this column is highly redundant for making any sort of prediction, you should make sure that values heavily related to this column are no already expressed in another column (e.g. if this column is a timestamp, make sure you don't have another column representing the exact same time in ISO datetime format).
+            The value is based in equal part on the Similarity Score and the Correlation Score.
+            """}
 
     def _compute_variability_score(self, stats, col_name):
         """
@@ -545,7 +576,11 @@ def _compute_variability_score(self, stats, col_name):
         else:
             variability_score = col_stats['value_distribution_score']/2
 
-        return {'variability_score': variability_score}
+        return {'variability_score': variability_score
+        ,'variability_score_description':"""
+        A high value for this score indicates the data in this column seems to be very variable, indicating a large possibility of some random noise affecting your data. This could mean that the values for this column are not collected or processed correctly.
+        The value is based in equal part on the Z Test based outliers score, the LOG based outlier score and the Value Distribution Score.
+        """}
 
 
     def _compute_data_quality_score(self, stats, col_name):
@@ -568,7 +603,10 @@ def _compute_data_quality_score(self, stats, col_name):
             quality_score += col_stats[score]
         quality_score = quality_score/len(scores)
 
-        return {'quality_score': quality_score}
+        return {'quality_score': quality_score
+        ,'quality_score_description':"""
+        The higher this score is, the lower the quality of a given column.
+        """}
 
     def _log_interesting_stats(self, stats):
         """
@@ -585,50 +623,89 @@ def _log_interesting_stats(self, stats):
                 self.log.warning('Column "{}" is considered of low quality, the scores that influenced this decission will be listed bellow')
                 if col_stats['duplicates_score'] > 0.5:
                     duplicates_percentage = col_stats['duplicates_percentage']
-                    self.log.warning(f'{duplicates_percentage}% of the values in column {col_name} seem to be repeated, this might indicate your data is of poor quality.')
-
-
+                    w = f'{duplicates_percentage}% of the values in column {col_name} seem to be repeated, this might indicate your data is of poor quality.'
+                    self.log.warning(w)
+                    col_stats['duplicates_score_warning'] = w
+                else:
+                    col_stats['duplicates_score_warning'] = None
+            else:
+                col_stats['duplicates_score_warning'] = None
+                
             #Compound scores
             if col_stats['consistency_score'] > 0.25:
-                self.log.warning(f'The values in column {col_name} rate poorly in terms of consistency. This means the data has too many empty values, values with a hard to determine type and duplicate values. Please see the detailed logs bellow for more info')
+                w = f'The values in column {col_name} rate poorly in terms of consistency. This means the data has too many empty values, values with a hard to determine type and duplicate values. Please see the detailed logs bellow for more info'
+                self.log.warning(w)
+                col_stats['consistency_score_warning'] = w
+            else:
+                col_stats['consistency_score_warning'] = None
 
             if col_stats['redundancy_score'] > 0.45:
-                self.log.warning(f'The data in the column {col_name} is likely somewhat redundant, any insight it can give us can already by deduced from your other columns. Please see the detailed logs bellow for more info')
+                w = f'The data in the column {col_name} is likely somewhat redundant, any insight it can give us can already by deduced from your other columns. Please see the detailed logs bellow for more info'
+                self.log.warning(w)
+                col_stats['redundancy_score_warning'] = w
+            else:
+                col_stats['redundancy_score_warning'] = None
 
             if col_stats['variability_score'] > 0.5:
-                self.log.warning(f'The data in the column {col_name} seems to have too contain too much noise/randomness based on the value variability. That is too say, the data is too unevenly distributed and has too many outliers. Please see the detailed logs bellow for more info.')
-
-
+                w = f'The data in the column {col_name} seems to have too contain too much noise/randomness based on the value variability. That is too say, the data is too unevenly distributed and has too many outliers. Please see the detailed logs bellow for more info.'
+                self.log.warning(w)
+                col_stats['variability_score_warning'] = w
+            else:
+                col_stats['variability_score_warning'] = None
 
             # Some scores are meaningful on their own, and the user should be warnned if they fall bellow a certain threshold
             if col_stats['empty_cells_score'] > 0.2:
                 empty_cells_percentage = col_stats['empty_percentage']
-                self.log.warning(f'{empty_cells_percentage}% of the values in column {col_name} are empty, this might indicate your data is of poor quality.')
+                w = f'{empty_cells_percentage}% of the values in column {col_name} are empty, this might indicate your data is of poor quality.'
+                self.log.warning(w)
+                col_stats['empty_cells_score_warning'] = w
+            else:
+                col_stats['empty_cells_score_warning'] = None
 
             if col_stats['data_type_distribution_score'] > 0.2:
                 #self.log.infoChart(stats[col_name]['data_type_dist'], type='list', uid='Dubious Data Type Distribution for column "{}"'.format(col_name))
                 percentage_of_data_not_of_principal_type = col_stats['data_type_distribution_score'] * 100
                 principal_data_type = col_stats['data_type']
-                self.log.warning(f'{percentage_of_data_not_of_principal_type}% of your data is not of type {principal_data_type}, which was detected to be the data type for column {col_name}, this might indicate your data is of poor quality.')
+                w = f'{percentage_of_data_not_of_principal_type}% of your data is not of type {principal_data_type}, which was detected to be the data type for column {col_name}, this might indicate your data is of poor quality.'
+                self.log.warning(w)
+                col_stats['data_type_distribution_score_warning'] = w
+            else:
+                col_stats['data_type_distribution_score_warning'] = None
 
             if 'z_test_based_outlier_score' in col_stats and col_stats['z_test_based_outlier_score'] > 0.3:
                 percentage_of_outliers = col_stats['z_test_based_outlier_score']*100
-                self.log.warning(f"""Column {col_name} has a very high amount of outliers, {percentage_of_outliers}% of your data is more than 3 standard deviations away from the mean, this means there might
-                be too much randomness in this column for us to make an accurate prediction based on it.""")
+                w = f"""Column {col_name} has a very high amount of outliers, {percentage_of_outliers}% of your data is more than 3 standard deviations away from the mean, this means there might
+                be too much randomness in this column for us to make an accurate prediction based on it."""
+                self.log.warning(w)
+                col_stats['z_test_based_outlier_score_warning'] = w
+            else:
+                col_stats['z_test_based_outlier_score_warning'] = None
 
             if 'lof_based_outlier_score' in col_stats and col_stats['lof_based_outlier_score'] > 0.3:
                 percentage_of_outliers = col_stats['percentage_of_log_based_outliers']
-                self.log.warning(f"""Column {col_name} has a very high amount of outliers, {percentage_of_outliers}% of your data doesn't fit closely in any cluster using the KNN algorithm (20n) to cluster your data, this means there might
-                be too much randomness in this column for us to make an accurate prediction based on it.""")
+                w = f"""Column {col_name} has a very high amount of outliers, {percentage_of_outliers}% of your data doesn't fit closely in any cluster using the KNN algorithm (20n) to cluster your data, this means there might
+                be too much randomness in this column for us to make an accurate prediction based on it."""
+                self.log.warning(w)
+                col_stats['lof_based_outlier_score_warning'] = w
+            else:
+                col_stats['lof_based_outlier_score_warning'] = None
 
             if col_stats['value_distribution_score'] > 0.8:
                 max_probability_key = col_stats['max_probability_key']
-                self.log.warning(f"""Column {col_name} is very biased towards the value {max_probability_key}, please make sure that the data in this column is correct !""")
+                w = f"""Column {col_name} is very biased towards the value {max_probability_key}, please make sure that the data in this column is correct !"""
+                self.log.warning(w)
+                col_stats['lvalue_distribution_score_warning'] = w
+            else:
+                col_stats['value_distribution_score_warning'] = None
 
             if col_stats['similarity_score'] > 0.5:
                 similar_percentage = col_stats['similarity_score'] * 100
                 similar_col_name = col_stats['most_similar_column_name']
-                self.log.warning(f'Column {col_name} and {similar_col_name} are {similar_percentage}% the same, please make sure these represent two distinct features of your data !')
+                w = f'Column {col_name} and {similar_col_name} are {similar_percentage}% the same, please make sure these represent two distinct features of your data !'
+                self.log.warning(w)
+                col_stats['lof_based_outlier_score_warning'] = w
+            else:
+                col_stats['similarity_score_warning'] = None
 
             '''
             if col_stats['correlation_score'] > 0.4:


Test patch: