Repository: DistrictDataLabs/yellowbrick

Issue:
Improve Parallel Coordinates
See #50 for a more detailed discussion. 

There are several things that need to be done to improve the parallel coordinates visualizer:
- [x] Benchmarking, it is currently SLOW
- [x] Add a `draw_instance` method so that instances can be added to the figure at any time
- [x] Add `DataFrame` support so that the visualizer can accept either an `ndarray` or a `DataFrame` as input.
- [x] Create a subclass `NormalizedParallelCoordinates` that normalizes the data to the space 0 to 1 before drawing the coordinates. 
- [x] Add subsampling of instances to reduce clutter and improve performance 
- [x] Add fast vs. slow drawing methods for performance
- [x] Add alpha so that you can see instances through the other lines 
- [ ] <strike>Create an optimization technique that reorders the columns such that the overlap of two instances by the same class is minimized</strike>

There are probably several more things and there are comments in the Parallel Coordinates codebase as well.



Discussion:
At PyCon 2018 sprints.  I'm going to take a pass at some of these items.
@bbengfort - should we also include an item to make the "sample" capability select random rows, as opposed to ordered rows.  For data sets with sorted targets, you don't end up with a true representative sample.
@thekylesaurus I think that is a great idea - [we were planning on making sample a uniform random sample](https://github.com/DistrictDataLabs/yellowbrick/blob/develop/yellowbrick/features/pcoords.py#L244) but the number of choices for how to conduct sampling led us to simply use `head`. If there is a sklearn-method that does sampling we could use that as a guide for how to treat the `sample` parameter, otherwise perhaps we could simply add a `shuffle=True` argument which would make `head` a uniform random sample? What do you think? 
@bbengfort - shuffle=True would definitely get the job done; but I'll check sklearn first to see if there is a different convention that we should inherit.

Resolving patch:
diff --git a/yellowbrick/features/pcoords.py b/yellowbrick/features/pcoords.py
--- a/yellowbrick/features/pcoords.py
+++ b/yellowbrick/features/pcoords.py
@@ -19,10 +19,13 @@
 ## Imports
 ##########################################################################
 
+from numpy import hstack, ones
+from numpy.random import RandomState
+
 from sklearn.preprocessing import MinMaxScaler, MaxAbsScaler
 from sklearn.preprocessing import Normalizer, StandardScaler
 
-from yellowbrick.utils import is_dataframe
+from yellowbrick.utils import is_dataframe, is_series
 from yellowbrick.features.base import DataVisualizer
 from yellowbrick.exceptions import YellowbrickTypeError, YellowbrickValueError
 from yellowbrick.style.colors import resolve_colors
@@ -149,6 +152,15 @@ class ParallelCoordinates(DataVisualizer):
         If int, specifies the maximum number of samples to display.
         If float, specifies a fraction between 0 and 1 to display.
 
+    random_state : int, RandomState instance or None
+        If int, random_state is the seed used by the random number generator;
+        If RandomState instance, random_state is the random number generator;
+        If None, the random number generator is the RandomState instance used
+        by np.random; only used if shuffle is True and sample < 1.0
+
+    shuffle : boolean, default: True
+        specifies whether sample is drawn randomly
+
     color : list or tuple, default: None
         optional list or tuple of colors to colorize lines
         Use either color to colorize the lines on a per class basis or
@@ -169,6 +181,12 @@ class ParallelCoordinates(DataVisualizer):
         Keyword arguments that are passed to the base class and may influence
         the visualization as defined in other Visualizers.
 
+    Attributes
+    --------
+
+    n_samples_ : int
+        number of samples included in the visualization object
+
     Examples
     --------
 
@@ -193,7 +211,7 @@ class ParallelCoordinates(DataVisualizer):
     }
 
     def __init__(self, ax=None, features=None, classes=None, normalize=None,
-                 sample=1.0, color=None, colormap=None, vlines=True,
+                 sample=1.0, random_state=None, shuffle=False, color=None, colormap=None, vlines=True,
                  vlines_kwds=None, **kwargs):
         super(ParallelCoordinates, self).__init__(
             ax, features, classes, color, colormap, **kwargs
@@ -225,40 +243,97 @@ def __init__(self, ax=None, features=None, classes=None, normalize=None,
             )
         self.sample = sample
 
+        # Set sample parameters
+        if isinstance(shuffle, bool):
+            self.shuffle = shuffle
+        else:
+            raise YellowbrickTypeError(
+                "`shuffle` parameter must be boolean"
+            )
+        if self.shuffle:
+            if (random_state is None) or isinstance(random_state, int):
+                self._rng = RandomState(random_state)
+            elif isinstance(random_state, RandomState):
+                self._rng = random_state
+            else:
+                raise YellowbrickTypeError(
+                    "`random_state` parameter must be None, int, or np.random.RandomState"
+                )
+        else:
+            self._rng = None
+
         # Visual Parameters
         self.show_vlines = vlines
         self.vlines_kwds = vlines_kwds or {
             'linewidth': 1, 'color': 'black'
         }
 
-    def draw(self, X, y, **kwargs):
+    def fit(self, X, y=None, **kwargs):
         """
-        Called from the fit method, this method creates the parallel
-        coordinates canvas and draws each instance and vertical lines on it.
+        The fit method is the primary drawing input for the
+        visualization since it has both the X and y data required for the
+        viz and the transform method does not.
+
+        Parameters
+        ----------
+        X : ndarray or DataFrame of shape n x m
+            A matrix of n instances with m features
+
+        y : ndarray or Series of length n
+            An array or series of target or class values
+
+        kwargs : dict
+            Pass generic arguments to the drawing method
+
+        Returns
+        -------
+        self : instance
+            Returns the instance of the transformer/visualizer
         """
+
         # Convert from dataframe
         if is_dataframe(X):
             X = X.as_matrix()
+        if is_series(y):
+            y = y.as_matrix()
 
-        # Choose a subset of samples
-        # TODO: allow selection of a random subset of samples instead of head
-
-        if isinstance(self.sample, int):
-            self.n_samples = min([self.sample, len(X)])
-        elif isinstance(self.sample, float):
-            self.n_samples = int(len(X) * self.sample)
-        X = X[:self.n_samples, :]
+        # Subsample
+        X, y = self._subsample(X, y)
 
         # Normalize
         if self.normalize is not None:
             X = self.normalizers[self.normalize].fit_transform(X)
 
-        # Get the shape of the data
+        # the super method calls draw and returns self 
+        super(ParallelCoordinates, self).fit(X, y, **kwargs)
+
+        # Fit always returns self.
+        return self
+
+    def draw(self, X, y, **kwargs):
+        """
+        Called from the fit method, this method creates the parallel
+        coordinates canvas and draws each instance and vertical lines on it.
+
+        Parameters
+        ----------
+        X : ndarray of shape n x m
+            A matrix of n instances with m features
+
+        y : ndarray of length n
+            An array or series of target or class values
+
+        kwargs : dict
+            Pass generic arguments to the drawing method
+
+        """
+
+        # Get shape of the sampled data
         nrows, ncols = X.shape
 
         # Create the xticks for each column
         # TODO: Allow the user to specify this feature
-        x = list(range(ncols))
+        increments = list(range(ncols))
 
         # Create the colors
         # TODO: Allow both colormap, listed colors, and palette definition
@@ -268,32 +343,35 @@ def draw(self, X, y, **kwargs):
         )
         colors = dict(zip(self.classes_, color_values))
 
-        # Track which labels are already in the legend
-        used_legends = set([])
-
         # TODO: Make this function compatible with DataFrames!
         # TODO: Make an independent function to allow addition of instances!
-        for idx, row in enumerate(X):
-            # TODO: How to map classmap to labels?
-            label = y[idx] # Get the label for the row
-            label = self.classes_[label]
-
-            if label not in used_legends:
-                used_legends.add(label)
-                self.ax.plot(x, row, color=colors[label], alpha=0.25, label=label, **kwargs)
-            else:
-                self.ax.plot(x, row, color=colors[label], alpha=0.25, **kwargs)
+
+        # Prepare to flatten data within each class
+        #   introduce separation between individual data points using None in x-values and arbitrary value (one) in
+        #   y-values
+        X_separated = hstack([X, ones((nrows, 1))])
+        increments_separated = increments.copy()
+        increments_separated.append(None)
+
+        # Plot each class
+        for label, color in sorted(colors.items()):
+            y_as_str = y.astype('str')  # must be consistent with class conversion in DataVisualizer.fit()
+            X_in_class = X_separated[y_as_str == label, :]
+            increments_in_class = increments_separated * len(X_in_class)
+            if len(X_in_class) > 0:
+                self.ax.plot(increments_in_class, X_in_class.flatten(),
+                             label=label, color=colors[label], alpha=0.25, linewidth=1, **kwargs)
 
         # Add the vertical lines
         # TODO: Make an independent function for override!
         if self.show_vlines:
-            for idx in x:
+            for idx in increments:
                 self.ax.axvline(idx, **self.vlines_kwds)
 
         # Set the limits
-        self.ax.set_xticks(x)
+        self.ax.set_xticks(increments)
         self.ax.set_xticklabels(self.features_)
-        self.ax.set_xlim(x[0], x[-1])
+        self.ax.set_xlim(increments[0], increments[-1])
 
     def finalize(self, **kwargs):
         """
@@ -313,3 +391,21 @@ def finalize(self, **kwargs):
         # Set the legend and the grid
         self.ax.legend(loc='best')
         self.ax.grid()
+
+    def _subsample(self, X, y):
+
+        # Choose a subset of samples
+        if isinstance(self.sample, int):
+            n_samples = min([self.sample, len(X)])
+        elif isinstance(self.sample, float):
+            n_samples = int(len(X) * self.sample)
+
+        if (n_samples < len(X)) and self.shuffle:
+            indices = self._rng.choice(len(X), n_samples, replace=False)
+        else:
+            indices = slice(n_samples)
+        X = X[indices, :]
+        y = y[indices]
+
+        self.n_samples_ = n_samples
+        return X, y
diff --git a/yellowbrick/utils/types.py b/yellowbrick/utils/types.py
--- a/yellowbrick/utils/types.py
+++ b/yellowbrick/utils/types.py
@@ -202,6 +202,27 @@ def is_dataframe(obj):
 isdataframe = is_dataframe
 
 
+def is_series(obj):
+    """
+    Returns True if the given object is a Pandas Series.
+
+    Parameters
+    ----------
+    obj: instance
+        The object to test whether or not is a Pandas Series.
+    """
+    try:
+        # This is the best method of type checking
+        from pandas import Series
+        return isinstance(obj, Series)
+    except ImportError:
+        # Pandas is not a dependency, so this is scary
+        return obj.__class__.__name__ == "Series"
+
+# Alias for closer name to isinstance and issubclass
+isseries = is_series
+
+
 def is_structured_array(obj):
     """
     Returns True if the given object is a Numpy Structured Array.


Test patch:
diff --git a/tests/baseline_images/test_features/test_pcoords/test_normalized_pcoords.png b/tests/baseline_images/test_features/test_pcoords/test_normalized_pcoords.png
Binary files a/tests/baseline_images/test_features/test_pcoords/test_normalized_pcoords.png and b/tests/baseline_images/test_features/test_pcoords/test_normalized_pcoords.png differ
diff --git a/tests/baseline_images/test_features/test_pcoords/test_parallel_coords.png b/tests/baseline_images/test_features/test_pcoords/test_parallel_coords.png
Binary files a/tests/baseline_images/test_features/test_pcoords/test_parallel_coords.png and b/tests/baseline_images/test_features/test_pcoords/test_parallel_coords.png differ
diff --git a/tests/test_features/test_pcoords.py b/tests/test_features/test_pcoords.py
--- a/tests/test_features/test_pcoords.py
+++ b/tests/test_features/test_pcoords.py
@@ -78,6 +78,32 @@ def test_pcoords_sample_int(self):
         visualizer = ParallelCoordinates(sample=10)
         visualizer.fit_transform(self.X, self.y)
 
+    def test_pcoords_sample_int_shuffle(self):
+        """
+        Assert no errors occur using integer 'sample' argument and shuffle, with different random_state args
+        """
+        visualizer = ParallelCoordinates(sample=3, shuffle=True)
+        visualizer.fit_transform(self.X, self.y)
+
+        visualizer = ParallelCoordinates(sample=3, shuffle=True, random_state=444)
+        visualizer.fit_transform(self.X, self.y)
+
+        visualizer = ParallelCoordinates(sample=3, shuffle=True, random_state=np.random.RandomState())
+        visualizer.fit_transform(self.X, self.y)
+
+    def test_pcoords_sample_int_shuffle_false(self):
+        """
+        Assert no errors occur using integer 'sample' argument and shuffle, with different random_state args
+        """
+        visualizer = ParallelCoordinates(sample=3, shuffle=False)
+        visualizer.fit_transform(self.X, self.y)
+
+        visualizer = ParallelCoordinates(sample=3, shuffle=False, random_state=444)
+        visualizer.fit_transform(self.X, self.y)
+
+        visualizer = ParallelCoordinates(sample=3, shuffle=False, random_state=np.random.RandomState())
+        visualizer.fit_transform(self.X, self.y)
+
     def test_pcoords_sample_int_invalid(self):
         """
         Negative int values should raise
@@ -92,6 +118,32 @@ def test_pcoords_sample_float(self):
         visualizer = ParallelCoordinates(sample=0.5)
         visualizer.fit_transform(self.X, self.y)
 
+    def test_pcoords_sample_float_shuffle(self):
+        """
+        Assert no errors occur using float 'sample' argument and shuffle, with different random_state args
+        """
+        visualizer = ParallelCoordinates(sample=0.5, shuffle=True)
+        visualizer.fit_transform(self.X, self.y)
+
+        visualizer = ParallelCoordinates(sample=0.5, shuffle=True, random_state=444)
+        visualizer.fit_transform(self.X, self.y)
+
+        visualizer = ParallelCoordinates(sample=0.5, shuffle=True, random_state=np.random.RandomState())
+        visualizer.fit_transform(self.X, self.y)
+
+    def test_pcoords_sample_float_shuffle_false(self):
+        """
+        Assert no errors occur using float 'sample' argument and shuffle, with different random_state args
+        """
+        visualizer = ParallelCoordinates(sample=0.5, shuffle=False)
+        visualizer.fit_transform(self.X, self.y)
+
+        visualizer = ParallelCoordinates(sample=0.5, shuffle=False, random_state=444)
+        visualizer.fit_transform(self.X, self.y)
+
+        visualizer = ParallelCoordinates(sample=0.5, shuffle=False, random_state=np.random.RandomState())
+        visualizer.fit_transform(self.X, self.y)
+
     def test_pcoords_sample_float_invalid(self):
         """
         Float values for 'sample' argument outside [0,1] should raise.
@@ -131,3 +183,65 @@ def test_integrated_pcoords(self):
         visualizer.fit_transform(X, y)
         visualizer.poof()
         self.assert_images_similar(visualizer)
+
+    @staticmethod
+    def test_static_subsample():
+        """
+        Assert output of subsampling method against expectations
+        """
+
+        ntotal = 100
+        ncols = 50
+
+        y = np.arange(ntotal)
+        X = np.ones((ntotal, ncols)) * y.reshape(ntotal, 1)
+
+        visualizer = ParallelCoordinates(sample=1.0, random_state=None, shuffle=False)
+        Xprime, yprime = visualizer._subsample(X, y)
+        assert np.array_equal(Xprime, X)
+        assert np.array_equal(yprime, y)
+
+        visualizer = ParallelCoordinates(sample=200, random_state=None, shuffle=False)
+        Xprime, yprime = visualizer._subsample(X, y)
+        assert np.array_equal(Xprime, X)
+        assert np.array_equal(yprime, y)
+
+        sample = 50
+        visualizer = ParallelCoordinates(sample=sample, random_state=None, shuffle=False)
+        Xprime, yprime = visualizer._subsample(X, y)
+        assert np.array_equal(Xprime, X[:sample, :])
+        assert np.array_equal(yprime, y[:sample])
+
+        sample = 50
+        visualizer = ParallelCoordinates(sample=sample, random_state=None, shuffle=True)
+        Xprime, yprime = visualizer._subsample(X, y)
+        assert np.array_equal(Xprime, X[yprime.flatten(), :])
+        assert len(Xprime) == sample
+        assert len(yprime) == sample
+
+        visualizer = ParallelCoordinates(sample=0.5, random_state=None, shuffle=False)
+        Xprime, yprime = visualizer._subsample(X, y)
+        assert np.array_equal(Xprime, X[:int(ntotal/2), :])
+        assert np.array_equal(yprime, y[:int(ntotal/2)])
+
+        sample = 0.5
+        visualizer = ParallelCoordinates(sample=sample, random_state=None, shuffle=True)
+        Xprime, yprime = visualizer._subsample(X, y)
+        assert np.array_equal(Xprime, X[yprime.flatten(), :])
+        assert len(Xprime) == ntotal * sample
+        assert len(yprime) == ntotal * sample
+
+        sample = 0.25
+        visualizer = ParallelCoordinates(sample=sample, random_state=444, shuffle=True)
+        Xprime, yprime = visualizer._subsample(X, y)
+        assert np.array_equal(Xprime, X[yprime.flatten(), :])
+        assert len(Xprime) == ntotal * sample
+        assert len(yprime) == ntotal * sample
+
+        sample = 0.99
+        visualizer = ParallelCoordinates(sample=sample, random_state=np.random.RandomState(), shuffle=True)
+        Xprime, yprime = visualizer._subsample(X, y)
+        assert np.array_equal(Xprime, X[yprime.flatten(), :])
+        assert len(Xprime) == ntotal * sample
+        assert len(yprime) == ntotal * sample
+
diff --git a/tests/test_utils/test_types.py b/tests/test_utils/test_types.py
--- a/tests/test_utils/test_types.py
+++ b/tests/test_utils/test_types.py
@@ -522,6 +522,40 @@ def test_not_is_dataframe(self, obj):
         """
         assert not is_dataframe(obj)
 
+    ##////////////////////////////////////////////////////////////////////
+    ## is_series testing
+    ##////////////////////////////////////////////////////////////////////
+
+    def test_series_alias(self):
+        """
+        Assert isseries aliases is_series
+        """
+        assert isseries is is_series
+
+    @pytest.mark.skipif(pd is None, reason="requires pandas")
+    def test_is_series(self):
+        """
+        Test that is_series works correctly
+        """
+        df = pd.Series([1, 2, 3])
+
+        assert is_series(df)
+
+    @pytest.mark.parametrize("obj", [
+        np.array([
+            (1,2.,'Hello'), (2,3.,"World")],
+            dtype=[('foo', 'i4'),('bar', 'f4'), ('baz', 'S10')]
+        ),
+        np.array([1,2,3]),
+        [1, 2, 3],
+    ],
+    ids=["structured array", "array", "list"])
+    def test_not_is_series(self, obj):
+        """
+        Test that is_series does not match non-dataframes
+        """
+        assert not is_series(obj)
+
     ##////////////////////////////////////////////////////////////////////
     ## is_structured_array testing
     ##////////////////////////////////////////////////////////////////////