Repository: pyjanitor-devs/pyjanitor

Issue:
[BUG]`sort_by_appearance` parameter fails when `dropna=True` in `pivot_longer`
# Brief Description

`sort_by_appearance` fails when `dropna=True` in pivot_longer

# System Information


- Operating system: macOS
- Python version (required): 3.9

# Minimally Reproducible Code

```py
url = 'https://raw.githubusercontent.com/tidyverse/tidyr/main/data-raw/billboard.csv'
df = pd.read_csv(url)
df.pivot_longer(column_names = 'wk*', dropna=True, sort_by_appearance=True)
```
-->

# Error Messages

<!-- If you get an error message, please paste it between the backticks here. -->

```
ValueError                                Traceback (most recent call last)
Input In [11], in <cell line: 1>()
----> 1 df.pivot_longer(column_names = 'wk*', dropna=True, sort_by_appearance=True)

File /opt/conda/lib/python3.9/site-packages/pandas_flavor/register.py:29, in register_dataframe_method.<locals>.inner.<locals>.AccessorMethod.__call__(self, *args, **kwargs)
     27 @wraps(method)
     28 def __call__(self, *args, **kwargs):
---> 29     return method(self._obj, *args, **kwargs)

File /workspaces/pyjanitor/janitor/functions/pivot.py:308, in pivot_longer(df, index, column_names, names_to, values_to, column_level, names_sep, names_pattern, names_transform, dropna, sort_by_appearance, ignore_index)
    279 df = df.copy()
    281 (
    282     df,
    283     index,
   (...)
    305     ignore_index,
    306 )
--> 308 return _computations_pivot_longer(
    309     df,
    310     index,
    311     column_names,
    312     names_to,
    313     values_to,
    314     names_sep,
    315     names_pattern,
    316     names_transform,
    317     dropna,
    318     sort_by_appearance,
    319     ignore_index,
    320 )

File /workspaces/pyjanitor/janitor/functions/pivot.py:622, in _computations_pivot_longer(df, index, column_names, names_to, values_to, names_sep, names_pattern, names_transform, dropna, sort_by_appearance, ignore_index)
    619 len_index = len(df)
    621 if all((names_pattern is None, names_sep is None)):
--> 622     return _base_melt(
    623         df=df,
    624         index=index,
    625         len_index=len_index,
    626         values_to=values_to,
    627         names_transform=names_transform,
    628         dropna=dropna,
    629         sort_by_appearance=sort_by_appearance,
    630         ignore_index=ignore_index,
    631     )
    633 if names_sep is not None:
    634     return _pivot_longer_names_sep(
    635         df=df,
    636         index=index,
   (...)
    644         ignore_index=ignore_index,
    645     )

File /workspaces/pyjanitor/janitor/functions/pivot.py:930, in _base_melt(df, index, len_index, values_to, names_transform, dropna, sort_by_appearance, ignore_index)
    927     values = df._values.ravel(order="F")
    928 values = {values_to: values}
--> 930 return _final_frame_longer(
    931     df=df,
    932     len_index=len_index,
    933     reps=reps,
    934     index=index,
    935     outcome=outcome,
    936     values=values,
    937     names_to=None,
    938     dropna=dropna,
    939     sort_by_appearance=sort_by_appearance,
    940     ignore_index=ignore_index,
    941 )

File /workspaces/pyjanitor/janitor/functions/pivot.py:1256, in _final_frame_longer(df, len_index, reps, index, outcome, values, names_to, dropna, sort_by_appearance, ignore_index)
   1253 df_index = None
   1255 if sort_by_appearance:
-> 1256     df = _sort_by_appearance_for_melt(df=df, len_index=len_index)
   1258 if ignore_index:
   1259     df.index = range(len(df))

File /workspaces/pyjanitor/janitor/functions/pivot.py:1193, in _sort_by_appearance_for_melt(df, len_index)
   1191 if not length_check:
   1192     index_sorter = np.arange(len(df))
-> 1193     index_sorter = np.reshape(index_sorter, (-1, len_index))
   1194     index_sorter = index_sorter.ravel(order="F")
   1195     df = df.take(index_sorter)

File <__array_function__ internals>:5, in reshape(*args, **kwargs)

File /opt/conda/lib/python3.9/site-packages/numpy/core/fromnumeric.py:298, in reshape(a, newshape, order)
    198 @array_function_dispatch(_reshape_dispatcher)
    199 def reshape(a, newshape, order='C'):
    200     """
    201     Gives a new shape to an array without changing its data.
    202 
   (...)
    296            [5, 6]])
    297     """
--> 298     return _wrapfunc(a, 'reshape', newshape, order=order)

File /opt/conda/lib/python3.9/site-packages/numpy/core/fromnumeric.py:57, in _wrapfunc(obj, method, *args, **kwds)
     54     return _wrapit(obj, method, *args, **kwds)
     56 try:
---> 57     return bound(*args, **kwds)
     58 except TypeError:
     59     # A TypeError occurs if the object does have such a method in its
     60     # class, but its signature is not identical to that of NumPy's. This
   (...)
     64     # Call _wrapit from within the except clause to ensure a potential
     65     # exception has a traceback chain.
     66     return _wrapit(obj, method, *args, **kwds)

ValueError: cannot reshape array of size 5307 into shape (317)
```



Discussion:

Resolving patch:
diff --git a/janitor/functions/pivot.py b/janitor/functions/pivot.py
--- a/janitor/functions/pivot.py
+++ b/janitor/functions/pivot.py
@@ -2,7 +2,7 @@
 from itertools import zip_longest, chain
 import re
 import warnings
-from typing import Optional, Pattern, Union, Callable, Generator
+from typing import Optional, Pattern, Union, Callable
 
 import numpy as np
 import pandas as pd
@@ -590,7 +590,7 @@ def _computations_pivot_longer(
     values_to: Union[list, str, None],
     names_sep: Union[str, Pattern],
     names_pattern: Union[list, tuple, str, Pattern],
-    names_transform: Union[str, Callable, dict],
+    names_transform: Union[str, Callable, dict, None],
     dropna: bool,
     sort_by_appearance: bool,
     ignore_index: bool,
@@ -616,13 +616,10 @@ def _computations_pivot_longer(
         column_names = pd.unique(column_names)
     df = df.loc[:, column_names]
 
-    len_index = len(df)
-
     if all((names_pattern is None, names_sep is None)):
         return _base_melt(
             df=df,
             index=index,
-            len_index=len_index,
             values_to=values_to,
             names_transform=names_transform,
             dropna=dropna,
@@ -634,7 +631,6 @@ def _computations_pivot_longer(
         return _pivot_longer_names_sep(
             df=df,
             index=index,
-            len_index=len_index,
             names_to=names_to,
             names_sep=names_sep,
             names_transform=names_transform,
@@ -648,7 +644,6 @@ def _computations_pivot_longer(
         return _pivot_longer_names_pattern_str(
             df=df,
             index=index,
-            len_index=len_index,
             names_to=names_to,
             names_pattern=names_pattern,
             names_transform=names_transform,
@@ -661,7 +656,6 @@ def _computations_pivot_longer(
     return _pivot_longer_names_pattern_sequence(
         df=df,
         index=index,
-        len_index=len_index,
         names_to=names_to,
         names_pattern=names_pattern,
         names_transform=names_transform,
@@ -675,10 +669,9 @@ def _computations_pivot_longer(
 def _pivot_longer_names_pattern_sequence(
     df: pd.DataFrame,
     index: Union[dict, None],
-    len_index: int,
     names_to: list,
     names_pattern: Union[list, tuple],
-    names_transform: Union[str, Callable, dict],
+    names_transform: Union[str, Callable, dict, None],
     dropna: bool,
     sort_by_appearance: bool,
     values_to: Union[str, list, tuple],
@@ -738,17 +731,9 @@ def _pivot_longer_names_pattern_sequence(
         arr = (entry for _, entry in arr.items())
         arr = zip(*zip_longest(*arr))
         arr = map(pd.Series, arr)
-        outcome = zip(outcome, arr)
-        if names_transform:
-            outcome = _names_transform(
-                names_transform, is_dataframe=False, values=outcome
-            )
-        outcome = {
-            name: arr._values.repeat(len_index) for name, arr in outcome
-        }
-
+        outcome = dict(zip(outcome, arr))
     else:
-        outcome = {}
+        outcome = None
         names_to = None
 
     mapping = pd.Series(mapping)
@@ -756,13 +741,13 @@ def _pivot_longer_names_pattern_sequence(
 
     df = _final_frame_longer(
         df=df,
-        len_index=len_index,
         reps=group_max,
         index=index,
         outcome=outcome,
         values=values,
         names_to=names_to,
         dropna=dropna,
+        names_transform=names_transform,
         sort_by_appearance=sort_by_appearance,
         ignore_index=ignore_index,
     )
@@ -773,7 +758,6 @@ def _pivot_longer_names_pattern_sequence(
 def _pivot_longer_names_pattern_str(
     df: pd.DataFrame,
     index: Union[dict, None],
-    len_index: int,
     names_to: list,
     names_pattern: Union[str, Pattern],
     names_transform: bool,
@@ -809,7 +793,6 @@ def _pivot_longer_names_pattern_str(
         return _base_melt(
             df=df,
             index=index,
-            len_index=len_index,
             values_to=values_to,
             names_transform=names_transform,
             dropna=dropna,
@@ -820,7 +803,6 @@ def _pivot_longer_names_pattern_str(
     return _pivot_longer_dot_value(
         df=df,
         index=index,
-        len_index=len_index,
         sort_by_appearance=sort_by_appearance,
         ignore_index=ignore_index,
         names_to=names_to,
@@ -833,7 +815,6 @@ def _pivot_longer_names_pattern_str(
 def _pivot_longer_names_sep(
     df: pd.DataFrame,
     index: Union[dict, None],
-    len_index: int,
     names_to: list,
     names_sep: Union[str, Pattern],
     values_to: str,
@@ -870,7 +851,6 @@ def _pivot_longer_names_sep(
         return _base_melt(
             df=df,
             index=index,
-            len_index=len_index,
             values_to=values_to,
             names_transform=names_transform,
             dropna=dropna,
@@ -881,7 +861,6 @@ def _pivot_longer_names_sep(
     return _pivot_longer_dot_value(
         df=df,
         index=index,
-        len_index=len_index,
         sort_by_appearance=sort_by_appearance,
         ignore_index=ignore_index,
         names_to=names_to,
@@ -894,7 +873,6 @@ def _pivot_longer_names_sep(
 def _base_melt(
     df: pd.DataFrame,
     index: Union[dict, None],
-    len_index: int,
     values_to: str,
     names_transform: Union[str, Callable, dict, None],
     dropna: bool,
@@ -907,14 +885,8 @@ def _base_melt(
 
     columns = df.columns
     reps = len(columns)
-    outcome = (
-        (name, columns.get_level_values(name)) for name in columns.names
-    )
-    if names_transform:
-        outcome = _names_transform(
-            names_transform, is_dataframe=False, values=outcome
-        )
-    outcome = {name: arr._values.repeat(len_index) for name, arr in outcome}
+    outcome = {name: columns.get_level_values(name) for name in columns.names}
+
     # offers a fast route
     # while still returning the underlying array
     # which could be an extension array
@@ -929,13 +901,13 @@ def _base_melt(
 
     return _final_frame_longer(
         df=df,
-        len_index=len_index,
         reps=reps,
         index=index,
         outcome=outcome,
         values=values,
         names_to=None,
         dropna=dropna,
+        names_transform=names_transform,
         sort_by_appearance=sort_by_appearance,
         ignore_index=ignore_index,
     )
@@ -944,7 +916,6 @@ def _base_melt(
 def _pivot_longer_dot_value(
     df: pd.DataFrame,
     index: Union[dict, None],
-    len_index: int,
     sort_by_appearance: bool,
     ignore_index: bool,
     names_to: list,
@@ -998,7 +969,7 @@ def _pivot_longer_dot_value(
     if len(mapping.columns) == 1:
         mapping = mapping.iloc[:, 0]
         values, group_max = _headers_single_series(df=df, mapping=mapping)
-        outcome = {}
+        outcome = None
 
     else:
         # For multiple columns, the labels in `.value`
@@ -1031,24 +1002,16 @@ def _pivot_longer_dot_value(
         values = _dict_from_grouped_names(df=df)
         outcome = indexer.loc[indexer[".value"] == outcome[0], other]
         group_max = len(outcome)
-        if names_transform:
-            outcome = _names_transform(
-                names_transform, is_dataframe=True, values=outcome
-            )
-        outcome = {
-            name: arr._values.repeat(len_index)
-            for name, arr in outcome.items()
-        }
 
     return _final_frame_longer(
         df=df,
-        len_index=len_index,
         reps=group_max,
         index=index,
         outcome=outcome,
         values=values,
         names_to=None,
         dropna=dropna,
+        names_transform=names_transform,
         sort_by_appearance=sort_by_appearance,
         ignore_index=ignore_index,
     )
@@ -1107,138 +1070,65 @@ def _dict_from_grouped_names(df: pd.DataFrame) -> dict:
     return {name: concat_compat(arr) for name, arr in outcome.items()}
 
 
-def _names_transform(
-    names_transform, is_dataframe, values
-) -> Union[pd.DataFrame, Generator]:
-    """
-    Convert column type using names_transform.
-
-    If is_dataframe (values is a DataFrame),
-    then astype is applied directly; otherwise an iteration.
-
-    Returns either a DataFrame or a generator.
-    """
-    if is_dataframe:
-        return values.astype(names_transform)
-    if isinstance(names_transform, dict):
-        outcome = (
-            (key, arr.astype(names_transform[key]))
-            for key, arr in values
-            if key in names_transform
-        )
-    else:
-        outcome = ((key, arr.astype(names_transform)) for key, arr in values)
-    return outcome
-
-
-def _sort_by_appearance_for_melt(
-    df: pd.DataFrame, len_index: int
-) -> pd.DataFrame:
-    """
-    This function sorts the resulting dataframe by appearance,
-    via the `sort_by_appearance` parameter in `computations_pivot_longer`.
-
-    A dataframe that is sorted by appearance is returned.
-    """
-
-    # explanation here to help future me :)
-
-    # if the height of the new dataframe
-    # is the same as the height of the original dataframe,
-    # then there is no need to sort by appearance
-    length_check = any((len_index == 1, len_index == len(df)))
-
-    # pd.melt flips the columns into vertical positions
-    # it `tiles` the index during the flipping
-    # example:
-
-    #          first last  height  weight
-    # person A  John  Doe     5.5     130
-    #        B  Mary   Bo     6.0     150
-
-    # melting the dataframe above yields:
-    # df.melt(['first', 'last'])
-
-    #   first last variable  value
-    # 0  John  Doe   height    5.5
-    # 1  Mary   Bo   height    6.0
-    # 2  John  Doe   weight  130.0
-    # 3  Mary   Bo   weight  150.0
-
-    # sort_by_appearance `untiles` the index
-    # and keeps all `John` before all `Mary`
-    # since `John` appears first in the original dataframe:
-
-    #   first last variable  value
-    # 0  John  Doe   height    5.5
-    # 1  John  Doe   weight  130.0
-    # 2  Mary   Bo   height    6.0
-    # 3  Mary   Bo   weight  150.0
-
-    # to get to this second form, which is sorted by appearance,
-    # get the lengths of the dataframe
-    # before and after it is melted
-    # for the example above, the length before melting is 2
-    # and after -> 4.
-    # reshaping allows us to track the original positions
-    # in the previous dataframe ->
-    # np.reshape([0,1,2,3], (-1, 2)) becomes
-    # array([[0, 1],
-    #        [2, 3]])
-    # ravel, with the Fortran order (`F`) ensures the John's are aligned
-    # before the Mary's -> [0, 2, 1, 3]
-    # the flattened array is then passed to `take`
-    if not length_check:
-        index_sorter = np.arange(len(df))
-        index_sorter = np.reshape(index_sorter, (-1, len_index))
-        index_sorter = index_sorter.ravel(order="F")
-        df = df.take(index_sorter)
-
-    return df
-
-
 def _final_frame_longer(
     df: pd.DataFrame,
-    len_index: int,
     reps: int,
     index: Union[dict, None],
     outcome: dict,
     values: dict,
     names_to: Union[list, None],
     dropna: bool,
+    names_transform: Union[str, Callable, dict, None],
     sort_by_appearance: bool,
     ignore_index: bool,
 ) -> pd.DataFrame:
     """
     Build final dataframe for pivot_longer.
     """
+    len_index = len(df)
     indexer = np.tile(np.arange(len_index), reps)
-    df_index = df.index[indexer]
-    if index:
-        index = {name: arr[indexer] for name, arr in index.items()}
-    else:
-        index = {}
 
-    if dropna and not names_to:
+    if (names_transform is not None) & (outcome is not None):
+        if isinstance(names_transform, dict):
+            outcome = {
+                key: arr.astype(names_transform[key], copy=False)
+                for key, arr in outcome.items()
+                if key in names_transform
+            }
+        else:
+            outcome = {
+                key: arr.astype(names_transform, copy=False)
+                for key, arr in outcome.items()
+            }
+    if outcome is not None:
+        outcome = {
+            name: arr._values.repeat(len_index)
+            for name, arr in outcome.items()
+        }
+    if dropna:
         if len(values) == 1:
             key = next(iter(values))
             any_nulls = pd.isna(values[key])
         else:
             any_nulls = [pd.isna(arr) for _, arr in values.items()]
-            bools = np.equal.reduce(any_nulls)
-            any_nulls = np.where(bools, any_nulls[0], False)
+            any_nulls = np.logical_and.reduce(any_nulls)
         if any_nulls.any():
             values = {name: arr[~any_nulls] for name, arr in values.items()}
-            df_index = df_index[~any_nulls]
-            if index:
-                index = {name: arr[~any_nulls] for name, arr in index.items()}
-            if outcome:
+            indexer = indexer[~any_nulls]
+            if outcome is not None:
                 outcome = {
                     name: arr[~any_nulls] for name, arr in outcome.items()
                 }
 
     any_nulls = None
-    bools = None
+
+    df_index = df.index[indexer]
+    if index:
+        index = {name: arr[indexer] for name, arr in index.items()}
+    else:
+        index = {}
+    if outcome is None:
+        outcome = {}
 
     df = {**index, **outcome, **values}
 
@@ -1253,7 +1143,9 @@ def _final_frame_longer(
     df_index = None
 
     if sort_by_appearance:
-        df = _sort_by_appearance_for_melt(df=df, len_index=len_index)
+        indexer = indexer.argsort(kind="stable")
+        df = df.take(indexer)
+    indexer = None
 
     if ignore_index:
         df.index = range(len(df))


Test patch:
diff --git a/tests/functions/test_pivot_longer.py b/tests/functions/test_pivot_longer.py
--- a/tests/functions/test_pivot_longer.py
+++ b/tests/functions/test_pivot_longer.py
@@ -3,6 +3,7 @@
 import pytest
 
 from pandas.testing import assert_frame_equal
+from pandas import NA
 
 
 @pytest.fixture
@@ -1293,3 +1294,39 @@ def test_preserve_extension_types():
     )
 
     assert_frame_equal(expected, actual)
+
+
+def test_dropna_sort_by_appearance():
+    """
+    Test output when `dropna=True` and
+    `sort_by_appearance=True`
+    """
+    # GH PR #1169, Issue #1168
+
+    treatments = dict(
+        id=range(1, 6),
+        A=("A", NA, "A", NA, NA),
+        A_date=(1, NA, 2, NA, NA),
+        B=(NA, "B", "B", NA, NA),
+        B_date=(NA, 3, 2, NA, NA),
+        other=(NA, NA, NA, "C", "D"),
+        other_date=(NA, NA, NA, 1, 5),
+    )
+    treatments = pd.DataFrame(treatments)
+    actual = treatments.pivot_longer(
+        index="id",
+        names_to=["date", "treatment"],
+        names_pattern=[".+date$", ".+"],
+        dropna=True,
+        sort_by_appearance=True,
+    )
+
+    expected = pd.lreshape(
+        treatments,
+        {
+            "treatment": ["A", "B", "other"],
+            "date": ["A_date", "B_date", "other_date"],
+        },
+    ).sort_values(["id", "treatment", "date"], ignore_index=True)
+
+    assert_frame_equal(actual, expected)