Repository: andersbogsnes/ml_tooling

Issue:
ENH: Add missing data visualization to dataset
We should be able to visualize the number of missing datapoints in our dataset

**Describe the solution you'd like**
An API such as 
```
data = Dataset()
data.plot.missing_data()
```

Discussion:
codecov[bot]: # [Codecov](https://codecov.io/gh/andersbogsnes/ml_tooling/pull/431?src=pr&el=h1) Report
> Merging [#431](https://codecov.io/gh/andersbogsnes/ml_tooling/pull/431?src=pr&el=desc) into [develop](https://codecov.io/gh/andersbogsnes/ml_tooling/commit/0da79efc1b0b3444890a07d537baab7214920284&el=desc) will **decrease** coverage by `0.63%`.
> The diff coverage is `n/a`.

[![Impacted file tree graph](https://codecov.io/gh/andersbogsnes/ml_tooling/pull/431/graphs/tree.svg?width=650&height=150&src=pr&token=ucKtL3BlxO)](https://codecov.io/gh/andersbogsnes/ml_tooling/pull/431?src=pr&el=tree)

```diff
@@             Coverage Diff             @@
##           develop     #431      +/-   ##
===========================================
- Coverage   100.00%   99.36%   -0.64%     
===========================================
  Files           54       55       +1     
  Lines         1391     1408      +17     
  Branches       160      162       +2     
===========================================
+ Hits          1391     1399       +8     
- Misses           0        8       +8     
- Partials         0        1       +1     
```


| [Impacted Files](https://codecov.io/gh/andersbogsnes/ml_tooling/pull/431?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [src/ml\_tooling/plots/missing\_data.py](https://codecov.io/gh/andersbogsnes/ml_tooling/pull/431/diff?src=pr&el=tree#diff-c3JjL21sX3Rvb2xpbmcvcGxvdHMvbWlzc2luZ19kYXRhLnB5) | `55.55% <0.00%> (ø)` | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/andersbogsnes/ml_tooling/pull/431?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/andersbogsnes/ml_tooling/pull/431?src=pr&el=footer). Last update [0da79ef...e3fc92f](https://codecov.io/gh/andersbogsnes/ml_tooling/pull/431?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fafd0a1..c71f4c2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@
 - The dataframe returned by `.make_prediction`now labels the columns in a more
 human friendly manner
 - Dataset now verifies that `load_training_data` and `load_prediction_data` do not return empty
+- Added a missing data visualization to `Dataset.plot`
 
 # v0.10.2
 - Fixed type inferences from data to sql in _load_data
diff --git a/docs/plotting.rst b/docs/plotting.rst
index 3b118ea..bdb9b45 100644
--- a/docs/plotting.rst
+++ b/docs/plotting.rst
@@ -139,4 +139,8 @@ Available Data Plots
     The size of the correlation can indicate important features, but can also
     hint at data leakage if the correlation is too strong.
 
+- :meth:`~DataVisualize.missing_data`:
+    Visualizes percentage of missing data for each column in the dataset. If no columns have missing data, will
+    simply show an empty plot.
+
 Continue to :doc:`transformers`
diff --git a/src/ml_tooling/data/base_data.py b/src/ml_tooling/data/base_data.py
index 88ab4d2..4af0d74 100644
--- a/src/ml_tooling/data/base_data.py
+++ b/src/ml_tooling/data/base_data.py
@@ -22,7 +22,10 @@ class Dataset(metaclass=abc.ABCMeta):
     train_y: Optional[DataType] = None
     train_x: Optional[pd.DataFrame] = None
     cached_data: Optional[pd.DataFrame] = None
-    plot: DataVisualize = None
+
+    @property
+    def plot(self):
+        return DataVisualize(self)
 
     def create_train_test(
         self,
@@ -70,7 +73,6 @@ class Dataset(metaclass=abc.ABCMeta):
     def x(self):
         if self._x is None:
             self._x, self._y = indexable(*self._load_training_data())
-            self.plot = DataVisualize(self)
         return self._x
 
     @x.setter
@@ -81,7 +83,6 @@ class Dataset(metaclass=abc.ABCMeta):
     def y(self):
         if self._y is None:
             self._x, self._y = indexable(*self._load_training_data())
-            self.plot = DataVisualize(self)
         return self._y
 
     @y.setter
diff --git a/src/ml_tooling/data/viz/data_viz.py b/src/ml_tooling/data/viz/data_viz.py
index f843d23..74c466f 100644
--- a/src/ml_tooling/data/viz/data_viz.py
+++ b/src/ml_tooling/data/viz/data_viz.py
@@ -1,8 +1,8 @@
-from typing import Optional
+from typing import Optional, Union
 
 from matplotlib.axes import Axes
 import matplotlib.pyplot as plt
-from ml_tooling.plots import plot_target_correlation
+from ml_tooling.plots import plot_target_correlation, plot_missing_data
 from ml_tooling.config import MPL_STYLESHEET
 
 
@@ -14,8 +14,8 @@ class DataVisualize:
         self,
         method: str = "spearman",
         ax: Optional[Axes] = None,
-        top_n: Optional[int] = None,
-        bottom_n: Optional[int] = None,
+        top_n: Optional[Union[int, float]] = None,
+        bottom_n: Optional[Union[int, float]] = None,
     ) -> Axes:
         """
         Plot the correlation between each feature
@@ -55,3 +55,40 @@ class DataVisualize:
                 top_n=top_n,
                 bottom_n=bottom_n,
             )
+
+    def missing_data(
+        self,
+        ax: Optional[Axes] = None,
+        top_n: Optional[Union[int, float]] = None,
+        bottom_n: Optional[Union[int, float]] = None,
+    ) -> Axes:
+        """
+        Plot number of missing data points per column. Sorted by number of missing values.
+
+        Also allows for selecting top_n/bottom_n number or percent of columns by passing an int
+        or float
+
+        Parameters
+        ----------
+
+        ax: plt.Axes
+            Matplotlib axes to draw the graph on. Creates a new one by default
+
+        top_n: int, float
+            If top_n is an integer, return top_n features.
+            If top_n is a float between (0, 1), return top_n percent features
+
+        bottom_n: int, float
+            If bottom_n is an integer, return bottom_n features.
+            If bottom_n is a float between (0, 1), return bottom_n percent features
+
+
+        Returns
+        -------
+        plt.Axes
+        """
+
+        with plt.style.context(MPL_STYLESHEET):
+            return plot_missing_data(
+                df=self.data.x, ax=ax, top_n=top_n, bottom_n=bottom_n
+            )
diff --git a/src/ml_tooling/plots/__init__.py b/src/ml_tooling/plots/__init__.py
index 99ce4c1..835261f 100644
--- a/src/ml_tooling/plots/__init__.py
+++ b/src/ml_tooling/plots/__init__.py
@@ -8,6 +8,7 @@ from .roc_auc import plot_roc_auc
 from .precision_recall_curve import plot_pr_curve
 from .learning_curve import plot_learning_curve
 from .validation_curve import plot_validation_curve
+from .missing_data import plot_missing_data
 
 __all__ = [
     "plot_confusion_matrix",
@@ -20,4 +21,5 @@ __all__ = [
     "plot_pr_curve",
     "plot_learning_curve",
     "plot_validation_curve",
+    "plot_missing_data",
 ]
diff --git a/src/ml_tooling/plots/missing_data.py b/src/ml_tooling/plots/missing_data.py
new file mode 100644
index 0000000..13b3370
--- /dev/null
+++ b/src/ml_tooling/plots/missing_data.py
@@ -0,0 +1,57 @@
+from typing import Optional, Union
+
+import pandas as pd
+from ml_tooling.plots.utils import _plot_barh
+import matplotlib.pyplot as plt
+
+
+def plot_missing_data(
+    df: pd.DataFrame,
+    ax: Optional[plt.Axes] = None,
+    top_n: Optional[Union[int, float]] = None,
+    bottom_n: Optional[Union[int, float]] = None,
+    **kwargs,
+) -> plt.Axes:
+    """
+    Plot number of missing data points per column. Sorted by number of missing values.
+
+    Also allows for selecting top_n/bottom_n number or percent of columns by passing an int or float
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+        Feature DataFrame to calculate missing values from
+
+    ax: plt.Axes
+        Matplotlib axes to draw the graph on. Creates a new one by default
+
+    top_n: int, float
+        If top_n is an integer, return top_n features.
+        If top_n is a float between (0, 1), return top_n percent features
+
+    bottom_n: int, float
+        If bottom_n is an integer, return bottom_n features.
+        If bottom_n is a float between (0, 1), return bottom_n percent features
+
+    Returns
+    -------
+    plt.Axes
+    """
+    if ax is None:
+        fig, ax = plt.subplots()
+
+    missing_data = df.isna().sum().sort_values().divide(len(df)).loc[lambda x: x > 0]
+
+    return _plot_barh(
+        values=missing_data.values,
+        label_names=missing_data.index,
+        add_label=True,
+        title=f"Misssing data per column",
+        y_label="Feature",
+        x_label="Percent Missing Data",
+        ax=ax,
+        top_n=top_n,
+        bottom_n=bottom_n,
+        is_percent=True,
+        **kwargs,
+    )
diff --git a/src/ml_tooling/plots/utils.py b/src/ml_tooling/plots/utils.py
index 830b6e7..7219b84 100644
--- a/src/ml_tooling/plots/utils.py
+++ b/src/ml_tooling/plots/utils.py
@@ -1,6 +1,7 @@
 from typing import Union, Optional
 
 import matplotlib.pyplot as plt
+from matplotlib.ticker import PercentFormatter
 from matplotlib.axes import Axes, np
 from ml_tooling.metrics.utils import _is_percent, _sort_values
 from ml_tooling.utils import DataType
@@ -47,6 +48,7 @@ def _plot_barh(
     ax: Optional[Axes] = None,
     top_n: Union[int, float] = None,
     bottom_n: Union[int, float] = None,
+    is_percent=False,
     **kwargs,
 ) -> Axes:
     """
@@ -80,6 +82,9 @@ def _plot_barh(
         If bottom_n is an integer, return bottom_n features
         If bottom_n is a float between 0 and 1, return bottom_n percent of features
 
+    is_percent: bool
+        Indicates that the x-value is a percentage between 0 and 1. Formats the plot accordingly
+
 
     Returns
     -------
@@ -87,7 +92,7 @@ def _plot_barh(
     """
 
     if ax is None:
-        fig, ax = plt.subplots()
+        fig, ax = plt.subplots(constrained_layout=True)
 
     if top_n:
         title = (
@@ -118,12 +123,17 @@ def _plot_barh(
 
     if add_label:
         for i, (x, y) in enumerate(_generate_text_labels(ax, horizontal=True)):
+
             ax.annotate(
-                f"{values[i]:.2f}",
+                f"{values[i]:.1%}" if is_percent else f"{values[i]:.2f}",
                 (x, y),
                 xytext=(5, 0),
                 textcoords="offset points",
                 va="center",
             )
 
+    if is_percent:
+        formatter = PercentFormatter(xmax=1)
+        ax.xaxis.set_major_formatter(formatter)
+    ax.margins(x=0.15)
     return ax


Test patch:
diff --git a/tests/test_visualize.py b/tests/test_visualize.py
index f62a532..c1620c2 100644
--- a/tests/test_visualize.py
+++ b/tests/test_visualize.py
@@ -608,3 +608,45 @@ class TestTargetCorrelation:
         test_ax = train_iris_dataset.plot.target_correlation(ax=ax)
         assert ax == test_ax
         plt.close()
+
+
+class TestMissingDataViz:
+    @pytest.fixture()
+    def missing_data(self, train_boston_dataset):
+        train_boston_dataset.x.iloc[:10, 0] = np.nan
+        return train_boston_dataset
+
+    @pytest.fixture()
+    def ax(self, missing_data):
+        return missing_data.plot.missing_data()
+
+    def test_missing_data_text_labels_are_correct(self, ax):
+        assert [text.get_text() for text in ax.texts] == ["2.0%"]
+
+    def test_ylabel_is_correct(self, ax):
+        assert ax.get_ylabel() == "Feature"
+
+    def test_xlabel_is_correct(self, ax):
+        assert ax.get_xlabel() == "Percent Missing Data"
+
+    def test_xticklabels_are_correct(self, ax):
+        # Must trigger rendering before labels are accessible
+        ax.figure.canvas.draw()
+        assert [text.get_text() for text in ax.get_xticklabels()] == [
+            "0.00%",
+            "0.50%",
+            "1.00%",
+            "1.50%",
+            "2.00%",
+            "2.50%",
+        ]
+
+    def test_missing_data_plots_can_be_given_an_ax(self, missing_data):
+        fig, ax = plt.subplots()
+        test_ax = missing_data.plot.missing_data(ax=ax)
+        assert ax == test_ax
+        plt.close()
+
+    def test_can_call_missing_data_with_no_missing_values(self, train_boston_dataset):
+        ax = train_boston_dataset.plot.missing_data()
+        assert ax.patches == []