Repository: lincc-frameworks/nested-pandas

Issue:
Display of a single nested file loaded from parquet hangs indefinitely
Displaying a single nestedFrame loaded from parquet file hangs indefinitely. 

Following suggestions from this [thread](https://discovery-alliance.slack.com/archives/C04610PQW9F/p1746976462640049) I look at print statements:

```table = pq.read_table("/sdf/data/rubin/shared/lsdb_commissioning/mock_dp1/object_collection/object_lc/dataset/Norder=3/Dir=0/Npix=562.parquet")
df = table.to_pandas() 
%timeit print(df);
```
gives 
> 29 ms ± 124 μs per loop

while doing 

```from nested_pandas import NestedFrame, read_parquet
nested_df = read_parquet("/sdf/data/rubin/shared/lsdb_commissioning/mock_dp1/object_collection/object_lc/dataset/Norder=3/Dir=0/Npix=562.parquet")
%timeit print(nested_df)
```
gives
> 70.2 ms ± 342 μs per loop

But just plainly trying to run
`nested_df`
in Jupyter notebook is took 11 minutes and 10 seconds.

Hints from issue comments:

Solution patch:
diff --git a/docs/reference/accessor.rst b/docs/reference/accessor.rst
index 3ef47ba..8f01a4d 100644
--- a/docs/reference/accessor.rst
+++ b/docs/reference/accessor.rst
@@ -17,6 +17,7 @@ Functions
 
     NestSeriesAccessor.to_lists
     NestSeriesAccessor.to_flat
+    NestSeriesAccessor.to_flatten_inner
     NestSeriesAccessor.with_field
     NestSeriesAccessor.with_flat_field
     NestSeriesAccessor.with_list_field
@@ -26,4 +27,3 @@ Functions
     NestSeriesAccessor.get_flat_index
     NestSeriesAccessor.get_flat_series
     NestSeriesAccessor.get_list_series
-
diff --git a/src/nested_pandas/nestedframe/core.py b/src/nested_pandas/nestedframe/core.py
index ae27307..01a74a7 100644
--- a/src/nested_pandas/nestedframe/core.py
+++ b/src/nested_pandas/nestedframe/core.py
@@ -131,9 +131,16 @@ def repack_row(chunk):
                 return None
             return chunk.to_html(max_rows=1, max_cols=5, show_dimensions=True, index=False, header=False)
 
+        # Handle sizing, trim html dataframe if output will be truncated
+        df_shape = self.shape  # grab original shape information for later
+        if pd.get_option("display.max_rows") is None or df_shape[0] > pd.get_option("display.max_rows"):
+            html_df = self.head(pd.get_option("display.min_rows") + 1)
+        else:
+            html_df = self.copy()
+
         # replace index to ensure proper behavior for duplicate index values
-        index_values = self.index
-        html_df = self.reset_index(drop=True)
+        index_values = html_df.index
+        html_df = html_df.reset_index(drop=True)
 
         # Apply repacking to all nested columns
         repr = html_df.style.format(
@@ -152,15 +159,17 @@ def map_true_index(index):
 
         # Recover some truncation formatting, limited to head truncation
         if pd.get_option("display.max_rows") is None:
+            # Just display header
             return repr.to_html(max_rows=0)
-        elif repr.data.shape[0] > pd.get_option("display.max_rows"):
+        elif df_shape[0] > pd.get_option("display.max_rows"):
+            # when over the max_rows threshold, display with truncation ("..." row at the end)
             html_repr = repr.to_html(max_rows=pd.get_option("display.min_rows"))
         else:
             # when under the max_rows threshold, display all rows (behavior of 0 here)
             html_repr = repr.to_html(max_rows=0)
 
         # Manually append dimensionality to a styler output
-        html_repr += f"{repr.data.shape[0]} rows x {repr.data.shape[1]} columns"
+        html_repr += f"{df_shape[0]} rows x {df_shape[1]} columns"
 
         return html_repr
 
diff --git a/src/nested_pandas/series/accessor.py b/src/nested_pandas/series/accessor.py
index 627abe7..47c458a 100644
--- a/src/nested_pandas/series/accessor.py
+++ b/src/nested_pandas/series/accessor.py
@@ -12,7 +12,7 @@
 from pandas.api.extensions import register_series_accessor
 
 from nested_pandas.series.dtype import NestedDtype
-from nested_pandas.series.packer import pack_sorted_df_into_struct
+from nested_pandas.series.packer import pack_flat, pack_sorted_df_into_struct
 from nested_pandas.series.utils import nested_types_mapper
 
 __all__ = ["NestSeriesAccessor"]
@@ -540,3 +540,97 @@ def clear(self) -> None:
         The reason is that we cannot delete all nested fields from the nested series.
         """
         raise NotImplementedError("Cannot delete fields from nested series")
+
+    def to_flatten_inner(self, field: str) -> pd.Series:
+        """Explode the nested inner field and return as a pd.Series
+
+        Works for the case of multiple nesting only, the field must represent
+        a nested series.
+
+        Each row of this Series is changed in the following way:
+        1. Each nested item in the given field is converted to a "flat" frame.
+          If a nested item contains fields that are also nested, those are
+          brought up as their own nested structures in the resulting "flat"
+          frame.
+        2. All items of other fields are repeated as many times as that frame
+          length.
+
+        Parameters
+        ----------
+        field : str
+            Inner field, must have NestedDtype.
+
+        Returns
+        -------
+        pd.Series
+            This series object, but with inner field exploded.
+
+        Examples
+        --------
+        >>> from nested_pandas import NestedFrame
+        >>> from nested_pandas.datasets import generate_data
+        >>> nf = generate_data(5, 2, seed=1).rename(columns={"nested": "inner"})
+        >>> # Assign a repeated ID to double-nest on
+        >>> nf["id"] = [0, 0, 0, 1, 1]
+        >>> nf
+                  a         b                                              inner  id
+        0  0.417022  0.184677  [{t: 8.38389, flux: 80.074457, band: 'r'}; …] ...   0
+        1  0.720324  0.372520  [{t: 13.70439, flux: 96.826158, band: 'g'}; …]...   0
+        2  0.000114  0.691121  [{t: 4.089045, flux: 31.342418, band: 'g'}; …]...   0
+        3  0.302333  0.793535  [{t: 17.562349, flux: 69.232262, band: 'r'}; …...   1
+        4  0.146756  1.077633  [{t: 0.547752, flux: 87.638915, band: 'g'}; …]...   1
+        >>> nf.inner.nest.to_flat()
+                   t       flux band
+        0    8.38389  80.074457    r
+        0   13.40935  89.460666    g
+        1   13.70439  96.826158    g
+        1   8.346096   8.504421    g
+        2   4.089045  31.342418    g
+        2  11.173797   3.905478    g
+        3  17.562349  69.232262    r
+        3   2.807739  16.983042    r
+        4   0.547752  87.638915    g
+        4    3.96203   87.81425    r
+        >>> # Create a dataframe with double-nested column "outer"
+        >>> dnf = NestedFrame.from_flat(nf, base_columns=[], on="id", name="outer")
+        >>> # Flat "inner" nested column.
+        >>> # This is like "concatenation" of the initial nf frame on duplicated `id` rows
+        >>> concated_nf_series = dnf["outer"].nest.to_flatten_inner("inner")
+        >>> concated_nf_series
+        id
+        0    [{t: 8.38389, flux: 80.074457, band: 'r', a: 0...
+        1    [{t: 17.562349, flux: 69.232262, band: 'r', a:...
+        Name: inner, dtype: nested<t: [double], flux: [double], band: [string], a: [double], b: [double]>
+        >>> concated_nf_series.nest.to_flat()  # doctest: +NORMALIZE_WHITESPACE
+                    t       flux band         a         b
+        id
+        0     8.38389  80.074457    r  0.417022  0.184677
+        0    13.40935  89.460666    g  0.417022  0.184677
+        0    13.70439  96.826158    g  0.720324   0.37252
+        0    8.346096   8.504421    g  0.720324   0.37252
+        0    4.089045  31.342418    g  0.000114  0.691121
+        0   11.173797   3.905478    g  0.000114  0.691121
+        1   17.562349  69.232262    r  0.302333  0.793535
+        1    2.807739  16.983042    r  0.302333  0.793535
+        1    0.547752  87.638915    g  0.146756  1.077633
+        1     3.96203   87.81425    r  0.146756  1.077633
+        """
+        if not isinstance(self._series.dtype.field_dtype(field), NestedDtype):
+            raise ValueError(
+                f"Field '{field}' dtype must be NestedDtype, got '{self._series.dtype.field_dtype(field)}'"
+            )
+
+        # Copy series and make an "ordinal" index
+        series = self._series.reset_index(drop=True)
+        # Get a flat representation of the field
+        inner = self[field]
+        # Embed all other fields into the nested inner field, so the only field is left
+        for other_field in self.fields:
+            if other_field == field:
+                continue
+            inner = inner.nest.with_filled_field(other_field, series.nest[other_field])
+        # Repack flat inner back to nested series
+        result = pack_flat(inner.nest.to_flat(), name=field)
+        # Restore index
+        result.index = self._series.index
+        return result


Test patch:
diff --git a/tests/nested_pandas/series/test_accessor.py b/tests/nested_pandas/series/test_accessor.py
index d85f82b..30356b6 100644
--- a/tests/nested_pandas/series/test_accessor.py
+++ b/tests/nested_pandas/series/test_accessor.py
@@ -1073,3 +1073,28 @@ def test_get_list_index():
     list_index = series.array.get_list_index()
     assert len(list_index) == series.nest.flat_length
     assert np.equal(list_index, [0, 0, 0, 0, 1, 1, 1, 1]).all()
+
+
+def test_to_flatten_inner():
+    """Test .nest.to_flatten_inner()"""
+    nf = generate_data(10, 2)
+    nf["a"] = nf["a"].astype(pd.ArrowDtype(pa.float64()))
+    nf["b"] = nf["b"].astype(pd.ArrowDtype(pa.float64()))
+    nf = nf.assign(id=np.repeat(np.r_[0:5], 2))
+    nf = nf.rename(columns={"nested": "inner"})
+    nnf = NestedFrame.from_flat(nf, base_columns=[], on="id", name="outer")
+
+    outer_flatten = nnf["outer"].nest.to_flatten_inner("inner")
+
+    assert_frame_equal(
+        nf.drop("inner", axis=1).join(nf["inner"].nest.to_flat()).set_index("id"),
+        outer_flatten.nest.to_flat(),
+        check_like=True,
+    )
+
+
+def test_to_flatten_outer_wrong_field():
+    """Test an exception is raised when .nest.to_flatten_inner() called for a wrong field."""
+    nf = generate_data(10, 2)
+    with pytest.raises(ValueError):
+        nf.nested.nest.to_flatten_inner("t")