Repository: arviz-devs/arviz

Issue:
KDE plot produces strange results
**Description**
arviz's `plot_kde` produces plot with y-axis that doesn't make sense.

**Reproduce**
https://gist.github.com/absolutelyNoWarranty/f417a052c4b8d05fe3da3b95542f8a0a

**Expected behavior**
The peak of the kde should be around 0.4 for the standard gaussian just like Pandas's kde plot.

**Additional context**
Version of `arviz`: 0.3.2


Discussion:
Thanks for the reproducible example - it made it easy to check on this.

It looks like the difference is due to a difference in the algorithm being used - pandas is doing something different at the edge of the data, but arviz stops plotting at the edge.  Rescaling the x-axis shows that the plots are quite similar. 

I do not know much about the algorithms for a KDE, but @ahartikainen might have some opinions about whether we want to pad out the left and right edges of the data with 0's.

![image](https://user-images.githubusercontent.com/2295568/52276857-43395080-2921-11e9-86b9-d9fea4d47572.png)

@absolutelyNoWarranty thanks for taking the time to report this issue.

My opinion on this issue is that ArviZ is doing the a good job here, Pandas/Scipy's KDE do not respect the boundaries. In this example the result is over-smoothing the tails. Is not right to plot the density at the tail if you do not have data at the tails, at least not for a tool like ArviZ, this will show information about the posterior that is not there, if you want nicer tails, then you need more samples. Of course, for other tools people might argue that as a KDE is a quick way to perform inference about the density of the data the result by Pandas is a better one given that it looks closer to the the _true_ distribution. But for ArviZ this will be misleading. 
Yeah, I added a WIP branch to manually set limits.

There is currently a scaling issue. (We normalize by sum, we probably need to normalize by integration.)

Resolving patch:
diff --git a/arviz/plots/kdeplot.py b/arviz/plots/kdeplot.py
--- a/arviz/plots/kdeplot.py
+++ b/arviz/plots/kdeplot.py
@@ -69,7 +69,7 @@ def plot_kde(
         Use `space` keyword (float) to control the position of the rugplot. The larger this number
         the lower the rugplot.
     contour_kwargs : dict
-        Keywords passed to the contourplot. Ignored for 1D KDE
+        Keywords passed to the contourplot. Ignored for 1D KDE.
     ax : matplotlib axes
     legend : bool
         Add legend to the figure. By default True.
@@ -176,7 +176,7 @@ def plot_kde(
         if cumulative:
             density_q = density
         else:
-            density_q = np.cumsum(density)
+            density_q = density.cumsum() / density.sum()
         fill_func = ax.fill_between
         fill_x, fill_y = x, density
         if rotated:
@@ -240,7 +240,7 @@ def plot_kde(
     return ax
 
 
-def _fast_kde(x, cumulative=False, bw=4.5):
+def _fast_kde(x, cumulative=False, bw=4.5, xmin=None, xmax=None):
     """Fast Fourier transform-based Gaussian kernel density estimate (KDE).
 
     The code was adapted from https://github.com/mfouesneau/faststats
@@ -254,6 +254,10 @@ def _fast_kde(x, cumulative=False, bw=4.5):
         Bandwidth scaling factor for the KDE. Should be larger than 0. The higher this number the
         smoother the KDE will be. Defaults to 4.5 which is essentially the same as the Scott's rule
         of thumb (the default rule used by SciPy).
+    xmin : float
+        Manually set lower limit.
+    xmax : float
+        Manually set upper limit.
 
     Returns
     -------
@@ -264,12 +268,21 @@ def _fast_kde(x, cumulative=False, bw=4.5):
     x = np.asarray(x, dtype=float)
     x = x[np.isfinite(x)]
     len_x = len(x)
-    xmin, xmax = np.min(x), np.max(x)
+    n_points = 200 if (xmin or xmax) is None else 500
+
+    if xmin is None:
+        xmin = np.min(x)
+    if xmax is None:
+        xmax = np.max(x)
+
+    assert np.min(x) >= xmin
+    assert np.max(x) <= xmax
 
     std_x = entropy(x - xmin) * bw
 
-    n_bins = min(int(len_x ** (1 / 3) * std_x * 2), 200)
-    grid = _histogram(x, n_bins)
+    n_bins = min(int(len_x ** (1 / 3) * std_x * 2), n_points)
+    d_x = (xmax - xmin) / (n_bins - 1)
+    grid = _histogram(x, n_bins, range_hist=(xmin, xmax))
 
     scotts_factor = len_x ** (-0.2)
     kern_nx = int(scotts_factor * 2 * np.pi * std_x)
@@ -278,18 +291,19 @@ def _fast_kde(x, cumulative=False, bw=4.5):
     npad = min(n_bins, 2 * kern_nx)
     grid = np.concatenate([grid[npad:0:-1], grid, grid[n_bins : n_bins - npad : -1]])
     density = convolve(grid, kernel, mode="same", method="direct")[npad : npad + n_bins]
+    norm_factor = len_x * d_x * (2 * np.pi * std_x ** 2 * scotts_factor ** 2) ** 0.5
 
-    density /= np.sum(density)
+    density /= norm_factor
 
     if cumulative:
-        density = np.cumsum(density)
+        density = density.cumsum() / density.sum()
 
     return density, xmin, xmax
 
 
 @conditional_jit
-def _histogram(x, n_bins):
-    grid, _ = np.histogram(x, bins=n_bins)
+def _histogram(x, n_bins, range_hist=None):
+    grid, _ = np.histogram(x, bins=n_bins, range=range_hist)
     return grid
 
 


Test patch:
diff --git a/arviz/tests/test_plots.py b/arviz/tests/test_plots.py
--- a/arviz/tests/test_plots.py
+++ b/arviz/tests/test_plots.py
@@ -2,6 +2,7 @@
 import os
 import matplotlib.pyplot as plt
 from pandas import DataFrame
+from scipy.stats import gaussian_kde
 import numpy as np
 import pytest
 import pymc3 as pm
@@ -24,6 +25,7 @@
     plot_violin,
     plot_compare,
     plot_kde,
+    _fast_kde,
     plot_khat,
     plot_hpd,
 )
@@ -665,3 +667,23 @@ def test_plot_compare_no_ic(models):
 def test_plot_hpd(models, model_fit, data, kwargs):
     obj = getattr(models, model_fit)
     plot_hpd(data["y"], obj["theta"], **kwargs)
+
+
+@pytest.mark.parametrize("limits", [(-10.0, 10.0), (-5, 5), (None, None)])
+def test_fast_kde_scipy(limits):
+    data = np.random.normal(0, 1, 1000)
+    if limits[0] is None:
+        x = np.linspace(data.min(), data.max(), 200)
+    else:
+        x = np.linspace(*limits, 500)
+    density = gaussian_kde(data).evaluate(x)
+    density_fast = _fast_kde(data, xmin=limits[0], xmax=limits[1])[0]
+
+    np.testing.assert_almost_equal(density_fast.sum(), density.sum(), 1)
+
+
+@pytest.mark.parametrize("limits", [(-10.0, 10.0), (-5, 5), (None, None)])
+def test_fast_kde_cumulative(limits):
+    data = np.random.normal(0, 1, 1000)
+    density_fast = _fast_kde(data, xmin=limits[0], xmax=limits[1], cumulative=True)[0]
+    np.testing.assert_almost_equal(round(density_fast[-1], 3), 1)