Repository: pyjanitor-devs/pyjanitor

Issue:
Complete function for missing combinations of data
# Brief Description

I would like to propose the ``complete`` function, similar to tidyr's ``complete`` function, to get missing combinations of data.

# Example API

```python

#code idea:
@pf.register_dataframe_method
def complete(df, combinations: ..., fill_values = dictionary):
    """
    Expose missing combinations of data in a dataframe.
    """
   #get current positions of the columns
   cols = df.columns
    combo = pd.MultiIndex.from_product([df.loc[:,col].unique() for col in combinations], names = combinations)
    df = (df.set_index(combinations)
             .reindex(combo)
             .reset_index()
             .reindex(columns = cols))
    if fill_values:
        df = df.fillna(fill_values)  
    return df

# example data:
df = pd.DataFrame({'group': [1, 2, 1],
                   'item_id': [1, 2, 2],
                   'item_name': ['a', 'b', 'b'],
                   'value1': [1, 2, 3],
                   'value2': [4, 5, 6]})

	group	item_id	  item_name  value1	value2
1	     1 	    1  	        a      1  	    4
2	     2 	    2  	        b      2  	    5
3	     1 	    2  	        b      3  	    6

df.complete(combinations = ("group", "item_id", "item_name"))

	group	item_id	item_name	value1	value2
0	    1	     1	     a	        1.0	4.0
1	    1	     1	     b	        NaN	NaN
2	    1	     2	     a	        NaN	NaN
3	    1	     2	     b	        3.0	6.0
4	    2	     1	     a	        NaN	NaN
5	    2	     1	     b	        NaN	NaN
6	    2	     2	     a	        NaN	NaN
7	    2	     2	     b	        2.0	5.0

df.complete(combinations = ("group", "item_id", "item_name"), fill_values = {"value1" : 0}))

	group	item_id	item_name	value1	value2
0	    1	     1	     a	        1.0	4.0
1	    1	     1	     b	        0.0	NaN
2	    1	     2	     a	        0.0	NaN
3	    1	     2	     b	        3.0	6.0
4	    2	     1	     a	        0.0	NaN
5	    2	     1	     b	        0.0	NaN
6	    2	     2	     a	        0.0	NaN
7	    2	     2	     b	        2.0	5.0
# more examples below
# ...
```



Discussion:

Resolving patch:
diff --git a/janitor/functions.py b/janitor/functions.py
--- a/janitor/functions.py
+++ b/janitor/functions.py
@@ -2,6 +2,7 @@
 
 import datetime as dt
 import inspect
+import itertools
 import re
 import unicodedata
 import warnings
@@ -33,6 +34,7 @@
 from .utils import (
     _check_instance,
     _clean_accounting_column,
+    _complete_groupings,
     _currency_column_to_numeric,
     _grid_computation,
     _replace_empty_string_with_none,
@@ -4387,3 +4389,157 @@ def groupby_topk(
     return df.groupby(groupby_column_name).apply(
         lambda d: d.sort_values(sort_column_name, **sort_values_kwargs).head(k)
     )
+
+
+@pf.register_dataframe_method
+def complete(
+    df: pd.DataFrame,
+    columns: List[Union[List, Tuple, Dict, str]],
+    fill_value: Optional[Dict] = None,
+) -> pd.DataFrame:
+    """
+    This function shows all possible combinations in a dataframe, including
+    the missing values.
+
+    This function is similar to tidyr's `complete` function.
+
+    Individual combinations or combinations with groupings are possible.
+
+    .. code-block:: python
+
+        import pandas as pd
+        import janitor as jn
+
+            Year      Taxon         Abundance
+        0   1999    Saccharina         4
+        1   2000    Saccharina         5
+        2   2004    Saccharina         2
+        3   1999     Agarum            1
+        4   2004     Agarum            8
+
+        Data Source - http://imachordata.com/2016/02/05/you-complete-me/
+
+        Note that Year 2000 and Agarum pairing is missing. Let's make it
+        explicit:
+
+        df.complete(columns = ['Year', 'Taxon'])
+
+           Year      Taxon     Abundance
+        0  1999     Agarum         1.0
+        1  1999     Saccharina     4.0
+        2  2000     Agarum         NaN
+        3  2000     Saccharina     5.0
+        4  2004     Agarum         8.0
+        5  2004     Saccharina     2.0
+
+        The null value can be replaced with the fill_value argument:
+
+        df.complete(columns = ['Year', 'Taxon'],
+                    fill_value={"Abundance":0})
+
+           Year      Taxon     Abundance
+        0  1999     Agarum         1.0
+        1  1999     Saccharina     4.0
+        2  2000     Agarum         0.0
+        3  2000     Saccharina     5.0
+        4  2004     Agarum         8.0
+        5  2004     Saccharina     2.0
+
+        What if we wanted the explicit missing values for all the years from
+        1999 to 2004? Easy - simply pass a dictionary paring the column name
+        with the new values :
+
+        df.complete(columns = [{"Year": range(df.Year.min(),
+                                              df.Year.max() + 1)},
+                                       "Taxon"],
+                    fill_value={"Abundance":0})
+
+        Year      Taxon     Abundance
+    0   1999     Agarum         1.0
+    1   1999    Saccharina      4.0
+    2   2000     Agarum         0.0
+    3   2000    Saccharina      5.0
+    4   2001     Agarum         0.0
+    5   2001    Saccharina      0.0
+    6   2002     Agarum         0.0
+    7   2002    Saccharina      0.0
+    8   2003     Agarum         0.0
+    9  2003     Saccharina      0.0
+    10  2004     Agarum         8.0
+    11  2004    Saccharina      2.0
+
+    Functional usage syntax:
+
+    .. code-block:: python
+
+        import pandas as pd
+        import janitor as jn
+
+        df = pd.DataFrame(...)
+        df = jn.complete(
+            df = df,
+            columns= [
+                column_label,
+                (column1, column2, ...),
+                {column1: new_values, ...}
+            ],
+            fill_value = None
+        )
+
+    Method chaining syntax:
+
+    .. code-block:: python
+
+        df = (
+            pd.DataFrame(...)
+            .complete(columns=[
+                column_label,
+                (column1, column2, ...),
+                {column1: new_values, ...},
+            ],
+            fill_value=None,
+        )
+
+
+    :param df: A pandas dataframe.
+    :param columns: This is a list containing the columns to be
+        completed. It could be column labels (string trype),
+        a list/tuple of column labels, or a dictionary that pairs
+        column labels with new values.
+    :param fill_value: Dictionary pairing the columns with the null replacement
+        value.
+    :returns: A pandas dataframe with modified column(s).
+    :raises: ValueError if `columns` is empty.
+    :raises: TypeError if `columns` is not a list.
+    :raises: ValueError if entry in `columns` is not a
+        str/dict/list/tuple.
+    :raises: ValueError if entry in `columns` is a dict/list/tuple
+        and is empty.
+    """
+    df_c = df.copy()
+    if not isinstance(columns, list):
+        raise TypeError("Columns should be in a list")
+    if not columns:
+        raise ValueError("columns cannot be empty")
+    # if there is no grouping within the list of columns :
+    if all(isinstance(column, str) for column in columns):
+        # Using sets gets more speed than say np.unique or drop_duplicates
+        reindex_columns = [set(df_c[item].array) for item in columns]
+        reindex_columns = itertools.product(*reindex_columns)
+        df_c = df_c.set_index(columns)
+
+    else:
+        df_c, reindex_columns = _complete_groupings(df_c, columns)
+
+    if df_c.index.has_duplicates:
+        reindex_columns = pd.DataFrame(
+            [], index=pd.Index(reindex_columns, names=columns)
+        )
+        df_c = df_c.join(reindex_columns, how="outer").reset_index()
+    else:
+        df_c = df_c.reindex(sorted(reindex_columns)).reset_index()
+
+    if fill_value:
+        df_c = df_c.fillna(fill_value)
+
+    return df_c
diff --git a/janitor/timeseries.py b/janitor/timeseries.py
--- a/janitor/timeseries.py
+++ b/janitor/timeseries.py
@@ -4,6 +4,7 @@
 
 import pandas as pd
 import pandas_flavor as pf
+
 from janitor import check
 
 
diff --git a/janitor/utils.py b/janitor/utils.py
--- a/janitor/utils.py
+++ b/janitor/utils.py
@@ -537,3 +537,65 @@ def _grid_computation(entry: Dict) -> pd.DataFrame:
             result = pd.DataFrame.join(first, rest).reset_index(drop=True)
 
     return result
+
+
+def _complete_groupings(df, list_of_columns):
+    # this collects all the columns as individual labels, which will be
+    # used to set the index of the dataframe
+    index_columns = []
+    # this will collect all the values associated with the respective
+    # columns, and used to reindex the dataframe, to get the complete
+    # pairings
+    reindex_columns = []
+    for item in list_of_columns:
+        if not isinstance(item, (str, dict, list, tuple)):
+            raise ValueError(
+                """Value must either be a column label, a list/tuple of columns or a
+                    dictionary where the keys are columns in the dataframe."""
+            )
+        if not item:
+            raise ValueError("grouping cannot be empty")
+        if isinstance(item, str):
+            reindex_columns.append(set(df[item].array))
+            index_columns.append(item)
+        else:
+            # this comes into play if we wish to input values that
+            # do not exist in the data, say years, or alphabets, or
+            # range of numbers
+            if isinstance(item, dict):
+                if len(item) > 1:
+                    index_columns.extend(item.keys())
+                else:
+                    index_columns.append(*item.keys())
+                    item_contents = [
+                        # convert scalars to iterables; this is necessary
+                        # when creating combinations with itertools' product
+                        [value]
+                        if isinstance(value, (int, float, str, bool))
+                        else value
+                        for key, value in item.items()
+                    ]
+                    reindex_columns.extend(item_contents)
+            else:
+                index_columns.extend(item)
+                item = (df[sub_column].array for sub_column in item)
+                item = set(zip(*item))
+                reindex_columns.append(item)
+
+    reindex_columns = product(*reindex_columns)
+    # A list comprehension, coupled with itertools chain.from_iterable
+    # would likely be faster; I fear that it may hamper readability with
+    # nested list comprehensions; as such, I chose the for loop method.
+    new_reindex_columns = []
+    for row in reindex_columns:
+        new_row = []
+        for cell in row:
+            if isinstance(cell, tuple):
+                new_row.extend(cell)
+            else:
+                new_row.append(cell)
+        new_reindex_columns.append(tuple(new_row))
+
+    df = df.set_index(index_columns)
+
+    return df, new_reindex_columns


Test patch:
diff --git a/tests/functions/test_complete.py b/tests/functions/test_complete.py
new file mode 100644
--- /dev/null
+++ b/tests/functions/test_complete.py
@@ -0,0 +1,322 @@
+import itertools
+
+import numpy as np
+import pandas as pd
+import pytest
+from pandas.testing import assert_frame_equal
+
+# from https://tidyr.tidyverse.org/reference/complete.html
+df = pd.DataFrame(
+    {
+        "group": [1, 2, 1],
+        "item_id": [1, 2, 2],
+        "item_name": ["a", "b", "b"],
+        "value1": [1, 2, 3],
+        "value2": [4, 5, 6],
+    }
+)
+
+columns = [
+    ["group", "item_id", "item_name"],
+    ["group", ("item_id", "item_name")],
+]
+
+expected_output = [
+    pd.DataFrame(
+        {
+            "group": [1, 1, 1, 1, 2, 2, 2, 2],
+            "item_id": [1, 1, 2, 2, 1, 1, 2, 2],
+            "item_name": ["a", "b", "a", "b", "a", "b", "a", "b"],
+            "value1": [1.0, np.nan, np.nan, 3.0, np.nan, np.nan, np.nan, 2.0],
+            "value2": [4.0, np.nan, np.nan, 6.0, np.nan, np.nan, np.nan, 5.0],
+        }
+    ),
+    pd.DataFrame(
+        {
+            "group": [1, 1, 2, 2],
+            "item_id": [1, 2, 1, 2],
+            "item_name": ["a", "b", "a", "b"],
+            "value1": [1.0, 3.0, np.nan, 2.0],
+            "value2": [4.0, 6.0, np.nan, 5.0],
+        }
+    ),
+]
+complete_parameters = [
+    (dataframe, columns, output)
+    for dataframe, (columns, output) in itertools.product(
+        [df], zip(columns, expected_output)
+    )
+]
+
+
+@pytest.mark.parametrize("df,columns,output", complete_parameters)
+def test_complete(df, columns, output):
+    """Test the complete function, with and without groupings."""
+    assert_frame_equal(df.complete(columns), output)
+
+
+# from http://imachordata.com/2016/02/05/you-complete-me/
+@pytest.fixture
+def df1():
+    return pd.DataFrame(
+        {
+            "Year": [1999, 2000, 2004, 1999, 2004],
+            "Taxon": [
+                "Saccharina",
+                "Saccharina",
+                "Saccharina",
+                "Agarum",
+                "Agarum",
+            ],
+            "Abundance": [4, 5, 2, 1, 8],
+        }
+    )
+
+
+def test_fill_value(df1):
+    """Test fill_value argument."""
+    output1 = pd.DataFrame(
+        {
+            "Year": [1999, 1999, 2000, 2000, 2004, 2004],
+            "Taxon": [
+                "Agarum",
+                "Saccharina",
+                "Agarum",
+                "Saccharina",
+                "Agarum",
+                "Saccharina",
+            ],
+            "Abundance": [1, 4.0, 0, 5, 8, 2],
+        }
+    )
+
+    result = df1.complete(
+        columns=["Year", "Taxon"], fill_value={"Abundance": 0}
+    )
+    assert_frame_equal(result, output1)
+
+
+def test_fill_value_all_years(df1):
+    """
+    Test the complete function accurately replicates for all the years
+    from 1999 to 2004.
+    """
+
+    output1 = pd.DataFrame(
+        {
+            "Year": [
+                1999,
+                1999,
+                2000,
+                2000,
+                2001,
+                2001,
+                2002,
+                2002,
+                2003,
+                2003,
+                2004,
+                2004,
+            ],
+            "Taxon": [
+                "Agarum",
+                "Saccharina",
+                "Agarum",
+                "Saccharina",
+                "Agarum",
+                "Saccharina",
+                "Agarum",
+                "Saccharina",
+                "Agarum",
+                "Saccharina",
+                "Agarum",
+                "Saccharina",
+            ],
+            "Abundance": [1.0, 4, 0, 5, 0, 0, 0, 0, 0, 0, 8, 2],
+        }
+    )
+
+    result = df1.complete(
+        columns=[
+            {"Year": range(df1.Year.min(), df1.Year.max() + 1)},
+            "Taxon",
+        ],
+        fill_value={"Abundance": 0},
+    )
+    assert_frame_equal(result, output1)
+
+
+def test_type_columns(df1):
+    """Raise error if columns is not a list object.'"""
+    with pytest.raises(TypeError):
+        df1.complete(columns="Year")
+
+
+def test_empty_columns(df1):
+    """Raise error if columns is empty'"""
+    with pytest.raises(ValueError):
+        df1.complete(columns=[])
+
+
+frame = pd.DataFrame(
+    {
+        "Year": [1999, 2000, 2004, 1999, 2004],
+        "Taxon": [
+            "Saccharina",
+            "Saccharina",
+            "Saccharina",
+            "Agarum",
+            "Agarum",
+        ],
+        "Abundance": [4, 5, 2, 1, 8],
+    }
+)
+wrong_columns = (
+    (frame, ["b", "Year"]),
+    (frame, [{"Yayay": range(7)}]),
+    (frame, ["Year", ["Abundant", "Taxon"]]),
+    (frame, ["Year", ("Abundant", "Taxon")]),
+)
+
+empty_sub_columns = [
+    (frame, ["Year", []]),
+    (frame, ["Year", {}]),
+    (frame, ["Year", ()]),
+    (frame, ["Year", set()]),
+]
+
+
+@pytest.mark.parametrize("frame,wrong_columns", wrong_columns)
+def test_wrong_columns(frame, wrong_columns):
+    """Test that KeyError is raised if wrong column is supplied."""
+    with pytest.raises(KeyError):
+        frame.complete(columns=wrong_columns)
+
+
+@pytest.mark.parametrize("frame,empty_sub_cols", empty_sub_columns)
+def test_empty_subcols(frame, empty_sub_cols):
+    """Raise ValueError for an empty container in columns'"""
+    with pytest.raises(ValueError):
+        frame.complete(columns=empty_sub_cols)
+
+
+# https://stackoverflow.com/questions/32874239/
+# how-do-i-use-tidyr-to-fill-in-completed-rows-within-each-value-of-a-grouping-var
+def test_grouping_first_columns():
+    """Test complete function when the first entry in columns is
+        a grouping."""
+
+    df2 = pd.DataFrame(
+        {
+            "id": [1, 2, 3],
+            "choice": [5, 6, 7],
+            "c": [9.0, np.nan, 11.0],
+            "d": [
+                pd.NaT,
+                pd.Timestamp("2015-09-30 00:00:00"),
+                pd.Timestamp("2015-09-29 00:00:00"),
+            ],
+        }
+    )
+    output2 = pd.DataFrame(
+        {
+            "id": [1, 1, 1, 2, 2, 2, 3, 3, 3],
+            "c": [9.0, 9.0, 9.0, np.nan, np.nan, np.nan, 11.0, 11.0, 11.0],
+            "d": [
+                pd.NaT,
+                pd.NaT,
+                pd.NaT,
+                pd.Timestamp("2015-09-30 00:00:00"),
+                pd.Timestamp("2015-09-30 00:00:00"),
+                pd.Timestamp("2015-09-30 00:00:00"),
+                pd.Timestamp("2015-09-29 00:00:00"),
+                pd.Timestamp("2015-09-29 00:00:00"),
+                pd.Timestamp("2015-09-29 00:00:00"),
+            ],
+            "choice": [5, 6, 7, 5, 6, 7, 5, 6, 7],
+        }
+    )
+    result = df2.complete(columns=[("id", "c", "d"), "choice"])
+    assert_frame_equal(result, output2)
+
+
+# https://stackoverflow.com/questions/48914323/tidyr-complete-cases-nesting-misunderstanding
+def test_complete_multiple_groupings():
+    """Test that `complete` gets the correct output for multiple groupings."""
+    df3 = pd.DataFrame(
+        {
+            "project_id": [1, 1, 1, 1, 2, 2, 2],
+            "meta": ["A", "A", "B", "B", "A", "B", "C"],
+            "domain1": ["d", "e", "h", "i", "d", "i", "k"],
+            "question_count": [3, 3, 3, 3, 2, 2, 2],
+            "tag_count": [2, 1, 3, 2, 1, 1, 2],
+        }
+    )
+
+    output3 = pd.DataFrame(
+        {
+            "project_id": [1, 1, 1, 1, 1, 2, 2, 2, 2, 2],
+            "meta": ["A", "A", "B", "B", "C", "A", "A", "B", "B", "C"],
+            "domain1": ["d", "e", "h", "i", "k", "d", "e", "h", "i", "k"],
+            "question_count": [3, 3, 3, 3, 3, 2, 2, 2, 2, 2],
+            "tag_count": [2.0, 1.0, 3.0, 2.0, 0.0, 1.0, 0.0, 0.0, 1.0, 2.0],
+        }
+    )
+
+    result = (
+        df3.complete(
+            columns=[("meta", "domain1"), ("project_id", "question_count")],
+            fill_value={"tag_count": 0},
+        )
+        # this part is not necessary for the test
+        # however, I wanted to match the result from Stack Overflow,
+        # hence the extra steps
+        .sort_values(["project_id", "meta"], ignore_index=True).reindex(
+            columns=df3.columns
+        )
+    )
+
+    assert_frame_equal(result, output3)
+
+
+# https://stackoverflow.com/questions/63541729/
+# pandas-how-to-include-all-columns-for-all-rows-although-value-is-missing-in-a-d
+# /63543164#63543164
+def test_duplicate_index():
+    """Test that the complete function works for duplicate index."""
+    df = pd.DataFrame(
+        {
+            "row": {
+                0: "21.08.2020",
+                1: "21.08.2020",
+                2: "21.08.2020",
+                3: "21.08.2020",
+                4: "22.08.2020",
+                5: "22.08.2020",
+                6: "22.08.2020",
+            },
+            "column": {0: "A", 1: "A", 2: "B", 3: "C", 4: "A", 5: "B", 6: "B"},
+            "value": {0: 43, 1: 36, 2: 36, 3: 28, 4: 16, 5: 40, 6: 34},
+        }
+    )
+
+    dup_expected_output = pd.DataFrame(
+        {
+            "row": [
+                "21.08.2020",
+                "21.08.2020",
+                "21.08.2020",
+                "21.08.2020",
+                "22.08.2020",
+                "22.08.2020",
+                "22.08.2020",
+                "22.08.2020",
+            ],
+            "column": ["A", "A", "B", "C", "A", "B", "B", "C"],
+            "value": [43.0, 36, 36, 28, 16, 40, 34, 0],
+        }
+    )
+
+    result = df.complete(columns=["row", "column"], fill_value={"value": 0})
+
+    assert_frame_equal(result, dup_expected_output)
diff --git a/tests/timeseries/test_fill_missing_timestamps.py b/tests/timeseries/test_fill_missing_timestamps.py
--- a/tests/timeseries/test_fill_missing_timestamps.py
+++ b/tests/timeseries/test_fill_missing_timestamps.py
@@ -1,7 +1,9 @@
-import pytest
-import pandas as pd
 from random import randint
-from janitor.timeseries import fill_missing_timestamps, _get_missing_timestamps
+
+import pandas as pd
+import pytest
+
+from janitor.timeseries import _get_missing_timestamps, fill_missing_timestamps
 
 
 # Random data for testing
diff --git a/tests/timeseries/test_sort_timestamps_monotonically.py b/tests/timeseries/test_sort_timestamps_monotonically.py
--- a/tests/timeseries/test_sort_timestamps_monotonically.py
+++ b/tests/timeseries/test_sort_timestamps_monotonically.py
@@ -1,6 +1,8 @@
-import pytest
-import pandas as pd
 from random import randint
+
+import pandas as pd
+import pytest
+
 import janitor  # noqa: F401
 import janitor.timeseries  # noqa: F401