Repository: lincc-frameworks/nested-pandas

Issue:
dtype bug when attempting to use dropna on a flat representation
**Bug report**
The following code fails. I'm attempting to call dropna on the flat representation of a nested dataframe, and providing a subset of columns to use for that operation. This operation fails with a confusing error, see below:
```
from nested_pandas import NestedFrame
import numpy as np
import pandas as pd

base = NestedFrame(data={"a": [1, 2, 3],
                          "b": [2, 4, 6]},
                         index=[0, 1, 2])
nested = pd.DataFrame(data={"c": [0, 2, 4, 1, np.NaN, 3, 1, 4, 1],
                            "d": [5, 4, 7, 5, 3, 1, 9, 3, 4]},
                         index=[0, 0, 0, 1, 1, 1, 2, 2, 2])
base = base.add_nested(nested, "nested")

base["nested"].nest.to_flat().dropna(subset="d")
```
```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[8], line 2
      1 import pyarrow as pa
----> 2 base["nested"].nest.to_flat().dropna(subset="d")

File ~/miniforge3/envs/ray310/lib/python3.10/site-packages/pandas/core/frame.py:6665, in DataFrame.dropna(self, axis, how, thresh, subset, inplace, ignore_index)
   6662     mask = count >= thresh
   6663 elif how == "any":
   6664     # faster equivalent to 'agg_obj.count(agg_axis) == self.shape[agg_axis]'
-> 6665     mask = notna(agg_obj).all(axis=agg_axis, bool_only=False)
   6666 elif how == "all":
   6667     # faster equivalent to 'agg_obj.count(agg_axis) > 0'
   6668     mask = notna(agg_obj).any(axis=agg_axis, bool_only=False)

File ~/miniforge3/envs/ray310/lib/python3.10/site-packages/pandas/core/frame.py:11615, in DataFrame.all(self, axis, bool_only, skipna, **kwargs)
  11607 @doc(make_doc("all", ndim=2))
  11608 def all(
  11609     self,
   (...)
  11613     **kwargs,
  11614 ) -> Series | bool:
> 11615     result = self._logical_func(
  11616         "all", nanops.nanall, axis, bool_only, skipna, **kwargs
  11617     )
  11618     if isinstance(result, Series):
  11619         result = result.__finalize__(self, method="all")

File ~/miniforge3/envs/ray310/lib/python3.10/site-packages/pandas/core/generic.py:12205, in NDFrame._logical_func(self, name, func, axis, bool_only, skipna, **kwargs)
  12202         obj = self._get_bool_data()
  12203     return obj._reduce_axis1(name, func, skipna=skipna)
> 12205 return self._reduce(
  12206     func,
  12207     name=name,
  12208     axis=axis,
  12209     skipna=skipna,
  12210     numeric_only=bool_only,
  12211     filter_type="bool",
  12212 )

File ~/miniforge3/envs/ray310/lib/python3.10/site-packages/pandas/core/frame.py:11552, in DataFrame._reduce(self, op, name, axis, skipna, numeric_only, filter_type, **kwds)
  11550 out = df._constructor_from_mgr(res, axes=res.axes).iloc[0]
  11551 if out_dtype is not None and out.dtype != "boolean":
> 11552     out = out.astype(out_dtype)
  11553 elif (df._mgr.get_dtypes() == object).any() and name not in ["any", "all"]:
  11554     out = out.astype(object)

File ~/miniforge3/envs/ray310/lib/python3.10/site-packages/pandas/core/generic.py:6625, in NDFrame.astype(self, dtype, copy, errors)
   6622                 raise
   6623         results.append(res_col)
-> 6625 elif is_extension_array_dtype(dtype) and self.ndim > 1:
   6626     # TODO(EA2D): special case not needed with 2D EAs
   6627     dtype = pandas_dtype(dtype)
   6628     if isinstance(dtype, ExtensionDtype) and all(
   6629         arr.dtype == dtype for arr in self._mgr.arrays
   6630     ):

File ~/miniforge3/envs/ray310/lib/python3.10/site-packages/pandas/core/dtypes/common.py:1328, in is_extension_array_dtype(arr_or_dtype)
   1326     return False
   1327 else:
-> 1328     return registry.find(dtype) is not None

File ~/miniforge3/envs/ray310/lib/python3.10/site-packages/pandas/core/dtypes/base.py:576, in Registry.find(self, dtype)
    574 for dtype_type in self.dtypes:
    575     try:
--> 576         return dtype_type.construct_from_string(dtype)
    577     except TypeError:
    578         pass

File ~/lincc/nested-pandas/src/nested_pandas/series/dtype.py:78, in NestedDtype.construct_from_string(cls, string)
     54 """Construct NestedDtype from a string representation.
     55 
     56 This works only for simple types, i.e. non-parametric pyarrow types.
   (...)
     75     are parametric pyarrow types.
     76 """
     77 if not string.startswith("nested<") or not string.endswith(">"):
---> 78     raise ValueError("Not a valid nested type string, expected 'nested<...>'")
     79 fields_str = string.removeprefix("nested<").removesuffix(">")
     81 field_strings = fields_str.split(", ")

ValueError: Not a valid nested type string, expected 'nested<...>'
```

**Before submitting**
Please check the following:

- [x] I have described the situation in which the bug arose, including what code was executed, information about my environment, and any applicable data others will need to reproduce the problem.
- [x] I have included available evidence of the unexpected behavior (including error messages, screenshots, and/or plots) as well as a descriprion of what I expected instead.
- [ ] If I have a solution in mind, I have provided an explanation and/or pseudocode and/or task list.

Discussion:

Patch:
diff --git a/src/nested_pandas/series/dtype.py b/src/nested_pandas/series/dtype.py
index 0d85409..3559b7b 100644
--- a/src/nested_pandas/series/dtype.py
+++ b/src/nested_pandas/series/dtype.py
@@ -70,32 +70,27 @@ class NestedDtype(ExtensionDtype):
 
         Raises
         ------
-        ValueError
+        TypeError
             If the string is not a valid nested type string or if the element types
             are parametric pyarrow types.
         """
         if not string.startswith("nested<") or not string.endswith(">"):
-            raise ValueError("Not a valid nested type string, expected 'nested<...>'")
+            raise TypeError("Not a valid nested type string, expected 'nested<...>'")
         fields_str = string.removeprefix("nested<").removesuffix(">")
 
         field_strings = fields_str.split(", ")
-        if len(field_strings) == 0:
-            raise ValueError(
-                "Not a valid nested type string, expected at least a single field inside "
-                "'nested<x: [type], ...>'"
-            )
 
         fields = {}
         for field_string in field_strings:
             try:
                 field_name, field_type = field_string.split(": ", maxsplit=1)
             except ValueError as e:
-                raise ValueError(
+                raise TypeError(
                     "Not a valid nested type string, expected 'nested<x: [type], ...>', got invalid field "
                     f"string '{field_string}'"
                 ) from e
             if not field_type.startswith("[") or not field_type.endswith("]"):
-                raise ValueError(
+                raise TypeError(
                     "Not a valid nested type string, expected 'nested<x: [type], ...>', got invalid field "
                     f"type string '{field_type}'"
                 )
@@ -105,7 +100,7 @@ class NestedDtype(ExtensionDtype):
             try:
                 pa_value_type = pa.type_for_alias(value_type)
             except ValueError as e:
-                raise ValueError(
+                raise TypeError(
                     f"Parsing pyarrow specific parameters in the string is not supported yet: {value_type}. "
                     "Please use NestedDtype() or NestedDtype.from_fields() instead."
                 ) from e


Test patch:
diff --git a/tests/nested_pandas/series/test_accessor.py b/tests/nested_pandas/series/test_accessor.py
index d3a736f..de65416 100644
--- a/tests/nested_pandas/series/test_accessor.py
+++ b/tests/nested_pandas/series/test_accessor.py
@@ -4,6 +4,7 @@ import pyarrow as pa
 import pytest
 from nested_pandas import NestedDtype
 from nested_pandas.series.ext_array import NestedExtensionArray
+from nested_pandas.series.packer import pack_flat
 from numpy.testing import assert_array_equal
 from pandas.testing import assert_frame_equal, assert_series_equal
 
@@ -503,3 +504,28 @@ def test___len__():
     series = pd.Series(struct_array, dtype=NestedDtype(struct_array.type), index=[0, 1])
 
     assert len(series.nest) == 2
+
+
+def test_to_flat_dropna():
+    """Test that to_flat() gives a valid dataframe, based on GH22
+
+    https://github.com/lincc-frameworks/nested-pandas/issues/22
+    """
+
+    flat = pd.DataFrame(
+        data={"c": [0.0, 2, 4, 1, np.NaN, 3, 1, 4, 1], "d": [5, 4, 7, 5, 3, 1, 9, 3, 4]},
+        index=[0, 0, 0, 1, 1, 1, 2, 2, 2],
+    )
+    nested = pack_flat(flat, name="nested")
+
+    new_flat = nested.nest.to_flat()
+    # .dropna() was failing in the issue report
+    filtered = new_flat.dropna(subset="c")
+
+    assert_frame_equal(
+        filtered,
+        pd.DataFrame(
+            data={"c": [0.0, 2, 4, 1, 3, 1, 4, 1], "d": [5, 4, 7, 5, 1, 9, 3, 4]},
+            index=[0, 0, 0, 1, 1, 2, 2, 2],
+        ),
+    )
diff --git a/tests/nested_pandas/series/test_dtype.py b/tests/nested_pandas/series/test_dtype.py
index 54c40bf..97aa32d 100644
--- a/tests/nested_pandas/series/test_dtype.py
+++ b/tests/nested_pandas/series/test_dtype.py
@@ -99,6 +99,30 @@ def test_name_vs_construct_from_string(fields):
     assert dtype == NestedDtype.construct_from_string(dtype.name)
 
 
+@pytest.mark.parametrize(
+    "s",
+    [
+        "float",  # not a nested type
+        "nested(f: [int64])",  # must be <> instead
+        "ts<in64>",  # 'ts' was a previous name, now we use 'nested'
+        "nested",  # no type specified
+        "nested<a: [int64]",  # missed closing bracket
+        "nested<>",  # no field specified
+        "nested<int64>",  # no field name specified
+        "nested<[int64]>",  # no field name specified
+        "nested<a:[int64]>",  # separator must be ": " with space
+        "nested<a: [int64],b: [float32]>",  # separator must be ", " with space
+        "nested<a: int64>",  # missed [] - nested list
+        "nested<a: [complex64]>",  # not an arrow type
+        "nested<a: [list<item: double>]>",  # complex arrow types are not supported
+    ],
+)
+def test_construct_from_string_raises(s):
+    """Test that we raise an error when constructing NestedDtype from invalid string."""
+    with pytest.raises(TypeError):
+        NestedDtype.construct_from_string(s)
+
+
 def test_construct_array_type():
     """Test that NestedDtype.construct_array_type() returns NestedExtensionArray."""
     assert NestedDtype.construct_array_type() is NestedExtensionArray