Repository: databricks/koalas

Issue:
Series Boxplot
Hey, in the Series Boxplot you show the top 1k absolute values.
However, may be some values which are fliers but don't have a big absolute value.
In my opinion, you should take the absolute value of the **difference**.

https://github.com/databricks/koalas/blob/b0e1c41389cd5ec09858cbf2a2f94ea38672a1bb/databricks/koalas/plot.py#L479-L492

I will illustrate another issue by a screen shot -

![image](https://user-images.githubusercontent.com/26877695/95206789-c9412500-07ef-11eb-8b4e-150da9df6f1c.png)

Thanks a lot :)


Discussion:
cc @HyukjinKwon 
@dvgodoy, sorry maybe dumb question. Why do we make the fliers as absolute values? I thought that we should keep their sign as is so they can be shown as proper outliners whether it is positive or negative but now it looks like they are all converted to positive absolute values? Reading the comment at https://github.com/databricks/koalas/pull/294#issuecomment-491607026, looks it was intentional.

@Noam2710, the absolute value of the difference sounds making sense. Would you mind opening a PR and going ahead?

@HyukjinKwon you and @Noam2710 are absolutely right - it was a silly mistake on my part when I implemented this. I got so concerned about limiting the number of fliers that I ended up introducing this error.
@dvgodoy, thanks for confirmation. Would you mind opening a PR to fix this when you find some time?

Resolving patch:
diff --git a/databricks/koalas/plot.py b/databricks/koalas/plot.py
--- a/databricks/koalas/plot.py
+++ b/databricks/koalas/plot.py
@@ -326,7 +326,7 @@ def _compute_plot_data(self):
         whiskers = KoalasBoxPlot._calc_whiskers(spark_column_name, outliers)
 
         if showfliers:
-            fliers = KoalasBoxPlot._get_fliers(spark_column_name, outliers)
+            fliers = KoalasBoxPlot._get_fliers(spark_column_name, outliers, whiskers[0])
         else:
             fliers = []
 
@@ -479,14 +479,17 @@ def _calc_whiskers(colname, outliers):
         return minmax.iloc[0][["min", "max"]].values
 
     @staticmethod
-    def _get_fliers(colname, outliers):
+    def _get_fliers(colname, outliers, min_val):
         # Filters only the outliers, should "showfliers" be True
         fliers_df = outliers.filter("`__{}_outlier`".format(colname))
 
         # If shows fliers, takes the top 1k with highest absolute values
+        # Here we normalize the values by subtracting the minimum value from
+        # each, and use absolute values.
+        order_col = F.abs(F.col("`{}`".format(colname)) - min_val.item())
         fliers = (
-            fliers_df.select(F.abs(F.col("`{}`".format(colname))).alias(colname))
-            .orderBy(F.desc("`{}`".format(colname)))
+            fliers_df.select(F.col("`{}`".format(colname)))
+            .orderBy(order_col)
             .limit(1001)
             .toPandas()[colname]
             .values


Test patch:
diff --git a/databricks/koalas/tests/test_series_plot.py b/databricks/koalas/tests/test_series_plot.py
--- a/databricks/koalas/tests/test_series_plot.py
+++ b/databricks/koalas/tests/test_series_plot.py
@@ -334,34 +334,38 @@ def check_box_plot(pser, kser, *args, **kwargs):
         )
 
     def test_box_summary(self):
-        kdf = self.kdf1
-        pdf = self.pdf1
-        k = 1.5
-
-        stats, fences = KoalasBoxPlot._compute_stats(kdf["a"], "a", whis=k, precision=0.01)
-        outliers = KoalasBoxPlot._outliers(kdf["a"], "a", *fences)
-        whiskers = KoalasBoxPlot._calc_whiskers("a", outliers)
-        fliers = KoalasBoxPlot._get_fliers("a", outliers)
-
-        expected_mean = pdf["a"].mean()
-        expected_median = pdf["a"].median()
-        expected_q1 = np.percentile(pdf["a"], 25)
-        expected_q3 = np.percentile(pdf["a"], 75)
-        iqr = expected_q3 - expected_q1
-        expected_fences = (expected_q1 - k * iqr, expected_q3 + k * iqr)
-        pdf["outlier"] = ~pdf["a"].between(fences[0], fences[1])
-        expected_whiskers = pdf.query("not outlier")["a"].min(), pdf.query("not outlier")["a"].max()
-        expected_fliers = pdf.query("outlier")["a"].values
-
-        self.assertEqual(expected_mean, stats["mean"])
-        self.assertEqual(expected_median, stats["med"])
-        self.assertEqual(expected_q1, stats["q1"] + 0.5)
-        self.assertEqual(expected_q3, stats["q3"] - 0.5)
-        self.assertEqual(expected_fences[0], fences[0] + 2.0)
-        self.assertEqual(expected_fences[1], fences[1] - 2.0)
-        self.assertEqual(expected_whiskers[0], whiskers[0])
-        self.assertEqual(expected_whiskers[1], whiskers[1])
-        self.assertEqual(expected_fliers, fliers)
+        def check_box_summary(kdf, pdf):
+            k = 1.5
+            stats, fences = KoalasBoxPlot._compute_stats(kdf["a"], "a", whis=k, precision=0.01)
+            outliers = KoalasBoxPlot._outliers(kdf["a"], "a", *fences)
+            whiskers = KoalasBoxPlot._calc_whiskers("a", outliers)
+            fliers = KoalasBoxPlot._get_fliers("a", outliers, whiskers[0])
+
+            expected_mean = pdf["a"].mean()
+            expected_median = pdf["a"].median()
+            expected_q1 = np.percentile(pdf["a"], 25)
+            expected_q3 = np.percentile(pdf["a"], 75)
+            iqr = expected_q3 - expected_q1
+            expected_fences = (expected_q1 - k * iqr, expected_q3 + k * iqr)
+            pdf["outlier"] = ~pdf["a"].between(fences[0], fences[1])
+            expected_whiskers = (
+                pdf.query("not outlier")["a"].min(),
+                pdf.query("not outlier")["a"].max(),
+            )
+            expected_fliers = pdf.query("outlier")["a"].values
+
+            self.assertEqual(expected_mean, stats["mean"])
+            self.assertEqual(expected_median, stats["med"])
+            self.assertEqual(expected_q1, stats["q1"] + 0.5)
+            self.assertEqual(expected_q3, stats["q3"] - 0.5)
+            self.assertEqual(expected_fences[0], fences[0] + 2.0)
+            self.assertEqual(expected_fences[1], fences[1] - 2.0)
+            self.assertEqual(expected_whiskers[0], whiskers[0])
+            self.assertEqual(expected_whiskers[1], whiskers[1])
+            self.assertEqual(expected_fliers, fliers)
+
+        check_box_summary(self.kdf1, self.pdf1)
+        check_box_summary(-self.kdf1, -self.pdf1)
 
     def test_kde_plot(self):
         def moving_average(a, n=10):