Repository: DistrictDataLabs/yellowbrick

Issue:
Incorporate kneed refactor in utils/kneed.py
yellowbrick has a port of some source code from the [kneed](https://github.com/arvkevi/kneed) package in `utils/kneed.py`. The `kneed` package was recently refactored with a [new release](https://github.com/arvkevi/kneed/releases/tag/v0.4.0).
I will create a PR to update `utils/kneed.py` to reflect the changes in `kneed`.

This [issue](https://github.com/DistrictDataLabs/yellowbrick/pull/813#issuecomment-513845068) raised by @Kautumn06 shows the elbow is "incorrectly" identified for the [example in the elbow documentation](https://www.scikit-yb.org/en/latest/api/cluster/elbow.html#).

The image shows knees identified in the example by incrementing `random_state` by 1 for 0-9999 in `make_blobs` for both the current and proposed (refactored) method. Could these `k<=7` runs be blobs with high variance (noisy clusters)? Should yb consider this separately and I create a new issue for this?

<img width="759" alt="Screen Shot 2019-07-22 at 10 08 37 PM" src="https://user-images.githubusercontent.com/9151717/61677416-8fa8fa80-accd-11e9-8a86-b873bcf3226f.png">



Discussion:

Resolving patch:
diff --git a/yellowbrick/utils/kneed.py b/yellowbrick/utils/kneed.py
--- a/yellowbrick/utils/kneed.py
+++ b/yellowbrick/utils/kneed.py
@@ -65,7 +65,7 @@ class KneeLocator(object):
        Sensitivity parameter that allows us to adjust how aggressive we want KneeLocator to 
        be when detecting "knees" or "elbows".
 
-    curve_nature : string, default: 'convace'
+    curve_nature : string, default: 'concave'
        A string that determines the nature of the elbow curve in which "knee" or "elbow" is 
        to be found.
 
@@ -77,9 +77,7 @@ class KneeLocator(object):
     -----
     The KneeLocator is implemented using the "knee point detection algorithm" which can be read at
     `<https://www1.icsi.berkeley.edu/~barath/papers/kneedle-simplex11.pdf>`
-
     """
-
     def __init__(self, x, y, S=1.0, curve_nature='concave', curve_direction='increasing'):
 
         # Raw Input
@@ -89,128 +87,145 @@ def __init__(self, x, y, S=1.0, curve_nature='concave', curve_direction='increas
         self.curve_direction = curve_direction
         self.N = len(self.x)
         self.S = S
+        self.all_knees = set()
+        self.all_norm_knees = set()
 
         # Step 1: fit a smooth line
         uspline = interpolate.interp1d(self.x, self.y)
-        self.Ds_x = np.linspace(np.min(self.x), np.max(self.x), self.N)
-        self.Ds_y = uspline(self.Ds_x)
+        self.x = np.array(x)
+        self.Ds_y = uspline(self.x)
 
         # Step 2: normalize values
-        self.xsn = self.__normalize(self.Ds_x)
-        self.ysn = self.__normalize(self.Ds_y)
-
-        # Step 3: Calculate difference curve
-        self.xd = self.xsn
-        if self.curve_nature == 'convex' and curve_direction == 'decreasing':
-            self.yd = self.ysn + self.xsn
-            self.yd = 1 - self.yd
-        elif self.curve_nature == 'concave' and curve_direction == 'decreasing':
-            self.yd = self.ysn + self.xsn
-        elif self.curve_nature == 'concave' and curve_direction == 'increasing':
-            self.yd = self.ysn - self.xsn
-        if self.curve_nature == 'convex' and curve_direction == 'increasing':
-            self.yd = abs(self.ysn - self.xsn)
+        self.x_normalized = self.__normalize(self.x)
+        self.y_normalized = self.__normalize(self.Ds_y)
+
+        # Step 3: Calculate the Difference curve
+        self.x_normalized, self.y_normalized = self.transform_xy(
+            self.x_normalized, self.y_normalized, self.curve_direction, self.curve_nature
+        )
+        # normalized difference curve
+        self.y_distance = self.y_normalized - self.x_normalized
+        self.x_distance = self.x_normalized.copy()
 
         # Step 4: Identify local maxima/minima
         # local maxima
-        self.xmx_idx = argrelextrema(self.yd, np.greater)[0]
-        self.xmx = self.xd[self.xmx_idx]
-        self.ymx = self.yd[self.xmx_idx]
+        self.maxima_inidices = argrelextrema(self.y_distance, np.greater)[0]
+        self.x_distance_maxima = self.x_distance[self.maxima_inidices]
+        self.y_distance_maxima = self.y_distance[self.maxima_inidices]
 
         # local minima
-        self.xmn_idx = argrelextrema(self.yd, np.less)[0]
-        self.xmn = self.xd[self.xmn_idx]
-        self.ymn = self.yd[self.xmn_idx]
+        self.minima_indices = argrelextrema(self.y_distance, np.less)[0]
+        self.x_distance_minima = self.x_distance[self.minima_indices]
+        self.y_distance_minima = self.y_distance[self.minima_indices]
 
         # Step 5: Calculate thresholds
-        self.Tmx = self.__threshold(self.ymx)
+        self.Tmx = self.y_distance_maxima - (self.S * np.abs(np.diff(self.x_normalized).mean()))
 
         # Step 6: find knee
-        self.knee, self.norm_knee, self.knee_x = self.find_knee()
+        self.find_knee()
+        self.knee, self.norm_knee = min(self.all_knees), min(self.all_norm_knees)
 
     @staticmethod
     def __normalize(a):
         """
         Normalizes an array.
-
         Parameters
         -----------
-        a : list 
+        a : list
            The array to normalize
         """
         return (a - min(a)) / (max(a) - min(a))
 
-    def __threshold(self, ymx_i):
-        """
-        Calculates the difference threshold for a
-        given difference local maximum.
-
-        Parameters
-        -----------
-        ymx_i : float 
-           The normalized y value of a local maximum.
-        """
-        return ymx_i - (self.S * np.diff(self.xsn).mean())
+    @staticmethod
+    def transform_xy(x, y, direction, curve):
+        """transform x and y to concave, increasing based on curve_direction and curve_nature"""
+        # convert elbows to knees
+        if curve == 'convex':
+            x = x.max() - x
+            y = y.max() - y
+        # flip decreasing functions to increasing
+        if direction == 'decreasing':
+            y = np.flip(y)
+
+        if curve == 'convex':
+            x = np.flip(x)
+            y = np.flip(y)
+
+        return x, y
 
     def find_knee(self, ):
-        """
-        Finds and returns the "knee"or "elbow" value, the normalized knee
-        value, and the x value where the knee is located.
-
-        """
-        if not self.xmx_idx.size:
+        """This function finds and sets the knee value and the normalized knee value. """
+        if not self.maxima_inidices.size:
             warning_message = \
                 'No "knee" or "elbow point" detected ' \
                 'This could be due to bad clustering, no '\
-                'actual clusters being formed etc.'        
-            warnings.warn(warning_message,YellowbrickWarning)
-            return None, None, None
-
-        mxmx_iter = np.arange(self.xmx_idx[0], len(self.xsn))
-        xmx_idx_iter = np.append(self.xmx_idx, len(self.xsn))
-
-        knee_, norm_knee_, knee_x = 0.0, 0.0, None
-        for mxmx_i, mxmx in enumerate(xmx_idx_iter):
-            # stopping criteria for exhasuting array
-            if mxmx_i == len(xmx_idx_iter) - 1:
+                'actual clusters being formed etc.'
+            warnings.warn(warning_message, YellowbrickWarning)
+            return None, None
+
+        # artificially place a local max at the last item in the x_distance array
+        self.maxima_inidices = np.append(self.maxima_inidices, len(self.x_distance) - 1)
+        self.minima_indices = np.append(self.minima_indices, len(self.x_distance) - 1)
+
+        # placeholder for which threshold region i is located in.
+        maxima_threshold_index = 0
+        minima_threshold_index = 0
+        # traverse the distance curve
+        for idx, i in enumerate(self.x_distance):
+            # reached the end of the curve
+            if i == 1.0:
                 break
-            # indices between maxima/minima
-            idxs = (mxmx_iter > xmx_idx_iter[mxmx_i]) * \
-                (mxmx_iter < xmx_idx_iter[mxmx_i + 1])
-            between_local_mx = mxmx_iter[np.where(idxs)]
-
-            for j in between_local_mx:
-                if j in self.xmn_idx:
-                    # reached a minima, x indices are unique
-                    # only need to check if j is a min
-                    if self.yd[j + 1] > self.yd[j]:
-                        self.Tmx[mxmx_i] = 0
-                        knee_x = None  # reset x where yd crossed Tmx
-                    elif self.yd[j + 1] <= self.yd[j]:
-                        warning_message="If this is a minima, " \
-                                        "how would you ever get here."
-                        warnings.warn(warning_message, YellowbrickWarning)
-                if self.yd[j] < self.Tmx[mxmx_i] or self.Tmx[mxmx_i] < 0:
-                    # declare a knee
-                    if not knee_x:
-                        knee_x = j
-                    knee_ = self.x[self.xmx_idx[mxmx_i]]
-                    norm_knee_ = self.xsn[self.xmx_idx[mxmx_i]]
-        return knee_, norm_knee_, knee_x
+            # values in distance curve are at or after a local maximum
+            if idx >= self.maxima_inidices[maxima_threshold_index]:
+                threshold = self.Tmx[maxima_threshold_index]
+                threshold_index = idx
+                maxima_threshold_index += 1
+            # values in distance curve are at or after a local minimum
+            if idx >= self.minima_indices[minima_threshold_index]:
+                threshold = 0.0
+                minima_threshold_index += 1
+            # Do not evaluate values in the distance curve before the first local maximum.
+            if idx < self.maxima_inidices[0]:
+                continue
+
+            # evaluate the threshold
+            if self.y_distance[idx] < threshold:
+                if self.curve_nature == 'convex':
+                    if self.curve_direction == 'decreasing':
+                        knee = self.x[threshold_index]
+                        self.all_knees.add(knee)
+                        norm_knee = self.x_normalized[threshold_index]
+                        self.all_norm_knees.add(norm_knee)
+                    else:
+                        knee = self.x[-(threshold_index + 1)]
+                        self.all_knees.add(knee)
+                        norm_knee = self.x_normalized[-(threshold_index + 1)]
+                        self.all_norm_knees.add(norm_knee)
+
+                elif self.curve_nature == 'concave':
+                    if self.curve_direction == 'decreasing':
+                        knee = self.x[-(threshold_index + 1)]
+                        self.all_knees.add(knee)
+                        norm_knee = self.x_normalized[-(threshold_index + 1)]
+                        self.all_norm_knees.add(norm_knee)
+                    else:
+                        knee = self.x[threshold_index]
+                        self.all_knees.add(knee)
+                        norm_knee = self.x_normalized[threshold_index]
+                        self.all_norm_knees.add(norm_knee)
 
     def plot_knee_normalized(self, ):
         """
-        Plots the normalized curve, the distance curve (xd, ysn) and the
+        Plots the normalized curve, the distance curve (x_distance, y_normalized) and the
         knee, if it exists.
-
         """
         import matplotlib.pyplot as plt
 
         plt.figure(figsize=(8, 8))
-        plt.plot(self.xsn, self.ysn)
-        plt.plot(self.xd, self.yd, 'r')
-        plt.xticks(np.arange(min(self.xsn), max(self.xsn) + 0.1, 0.1))
-        plt.yticks(np.arange(min(self.xd), max(self.ysn) + 0.1, 0.1))
+        plt.plot(self.x_normalized, self.y_normalized,)
+        plt.plot(self.x_distance, self.y_distance, 'r')
+        plt.xticks(np.arange(self.x_normalized.min(), self.x_normalized.max() + 0.1, 0.1))
+        plt.yticks(np.arange(self.y_distance.min(), self.y_normalized.max() + 0.1, 0.1))
 
         plt.vlines(self.norm_knee, plt.ylim()[0], plt.ylim()[1])
 
@@ -226,7 +241,6 @@ def plot_knee(self, ):
         plt.vlines(self.knee, plt.ylim()[0], plt.ylim()[1])
 
     # Niceties for users working with elbows rather than knees
-
     @property
     def elbow(self):
         return self.knee
@@ -236,7 +250,9 @@ def norm_elbow(self):
         return self.norm_knee
 
     @property
-    def elbow_x(self):
-        return self.knee_x
-
+    def all_elbows(self):
+        return self.all_knees
 
+    @property
+    def all_norm_elbows(self):
+        return self.all_norm_knees


Test patch: