Repository: lincc-frameworks/nested-pandas

Issue:
html_repr doesn't handle dropped dataframes
**Bug report**
Through filtering steps, we can get empty nested dataframes. These are replaced in the materialized view with just a NoneType, so the _repr_html code will fail when trying to call a to_html method from the nonetype. This is the reproducing code:
```
import nested_pandas as npd
import pandas as pd
base = npd.NestedFrame(data={"a": [3, 2, 1], "b": [2, 3, 6]}, index=[0, 1, 2])
nested = pd.DataFrame(
    data={"c": [0, 2, 5, 1, 4, 3, 1, 4, 1], "d": [5, 4, 7, 5, 3, 1, 9, 3, 4]},
    index=[0, 0, 0, 1, 1, 1, 2, 2, 2],
)
ndf = base.add_nested(nested, "nested")

ndf.query("nested.c>4")
```
And this is the error:
```
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
File [~/.virtualenvs/lsdb/lib/python3.12/site-packages/IPython/core/formatters.py:406](http://localhost:8888/~/.virtualenvs/lsdb/lib/python3.12/site-packages/IPython/core/formatters.py#line=405), in BaseFormatter.__call__(self, obj)
    404     method = get_real_method(obj, self.print_method)
    405     if method is not None:
--> 406         return method()
    407     return None
    408 else:

File [~/lincc/nested-pandas/src/nested_pandas/nestedframe/core.py:319](http://localhost:8888/~/lincc/nested-pandas/src/nested_pandas/nestedframe/core.py#line=318), in NestedFrame._repr_html_(self)
    316     html_repr = repr.to_html(max_rows=pd.get_option("display.min_rows"))
    317 else:
    318     # when under the max_rows threshold, display all rows (behavior of 0 here)
--> 319     html_repr = repr.to_html(max_rows=0)
    321 # Manually append dimensionality to a styler output
    322 html_repr += f"{repr.data.shape[0]} rows x {repr.data.shape[1]} columns"

File [~/.virtualenvs/lsdb/lib/python3.12/site-packages/pandas/io/formats/style.py:1345](http://localhost:8888/~/.virtualenvs/lsdb/lib/python3.12/site-packages/pandas/io/formats/style.py#line=1344), in Styler.to_html(self, buf, table_uuid, table_attributes, sparse_index, sparse_columns, bold_headers, caption, max_rows, max_columns, encoding, doctype_html, exclude_styles, **kwargs)
   1342     obj.set_caption(caption)
   1344 # Build HTML string..
-> 1345 html = obj._render_html(
   1346     sparse_index=sparse_index,
   1347     sparse_columns=sparse_columns,
   1348     max_rows=max_rows,
   1349     max_cols=max_columns,
   1350     exclude_styles=exclude_styles,
   1351     encoding=encoding or get_option("styler.render.encoding"),
   1352     doctype_html=doctype_html,
   1353     **kwargs,
   1354 )
   1356 return save_to_buffer(
   1357     html, buf=buf, encoding=(encoding if buf is not None else None)
   1358 )

File [~/.virtualenvs/lsdb/lib/python3.12/site-packages/pandas/io/formats/style_render.py:204](http://localhost:8888/~/.virtualenvs/lsdb/lib/python3.12/site-packages/pandas/io/formats/style_render.py#line=203), in StylerRenderer._render_html(self, sparse_index, sparse_columns, max_rows, max_cols, **kwargs)
    192 def _render_html(
    193     self,
    194     sparse_index: bool,
   (...)
    198     **kwargs,
    199 ) -> str:
    200     """
    201     Renders the ``Styler`` including all applied styles to HTML.
    202     Generates a dict with necessary kwargs passed to jinja2 template.
    203     """
--> 204     d = self._render(sparse_index, sparse_columns, max_rows, max_cols, "&nbsp;")
    205     d.update(kwargs)
    206     return self.template_html.render(
    207         **d,
    208         html_table_tpl=self.template_html_table,
    209         html_style_tpl=self.template_html_style,
    210     )

File [~/.virtualenvs/lsdb/lib/python3.12/site-packages/pandas/io/formats/style_render.py:187](http://localhost:8888/~/.virtualenvs/lsdb/lib/python3.12/site-packages/pandas/io/formats/style_render.py#line=186), in StylerRenderer._render(self, sparse_index, sparse_columns, max_rows, max_cols, blank)
    183         self.ctx_index[(r + ctx_len, c)] = v
    185     ctx_len += len(concatenated.index)
--> 187 d = self._translate(
    188     sparse_index, sparse_columns, max_rows, max_cols, blank, dxs
    189 )
    190 return d

File [~/.virtualenvs/lsdb/lib/python3.12/site-packages/pandas/io/formats/style_render.py:335](http://localhost:8888/~/.virtualenvs/lsdb/lib/python3.12/site-packages/pandas/io/formats/style_render.py#line=334), in StylerRenderer._translate(self, sparse_index, sparse_cols, max_rows, max_cols, blank, dxs)
    329 self.cellstyle_map: DefaultDict[tuple[CSSPair, ...], list[str]] = defaultdict(
    330     list
    331 )
    332 self.cellstyle_map_index: DefaultDict[
    333     tuple[CSSPair, ...], list[str]
    334 ] = defaultdict(list)
--> 335 body: list = self._translate_body(idx_lengths, max_rows, max_cols)
    336 d.update({"body": body})
    338 ctx_maps = {
    339     "cellstyle": "cellstyle_map",
    340     "cellstyle_index": "cellstyle_map_index",
    341     "cellstyle_columns": "cellstyle_map_columns",
    342 }  # add the cell_ids styles map to the render dictionary in right format

File [~/.virtualenvs/lsdb/lib/python3.12/site-packages/pandas/io/formats/style_render.py:629](http://localhost:8888/~/.virtualenvs/lsdb/lib/python3.12/site-packages/pandas/io/formats/style_render.py#line=628), in StylerRenderer._translate_body(self, idx_lengths, max_rows, max_cols)
    621     if self._check_trim(
    622         visible_row_count,
    623         max_rows,
    624         body,
    625         "row",
    626     ):
    627         break
--> 629     body_row = self._generate_body_row(
    630         (r, row_tup, rlabels), max_cols, idx_lengths
    631     )
    632     body.append(body_row)
    633 return body

File [~/.virtualenvs/lsdb/lib/python3.12/site-packages/pandas/io/formats/style_render.py:826](http://localhost:8888/~/.virtualenvs/lsdb/lib/python3.12/site-packages/pandas/io/formats/style_render.py#line=825), in StylerRenderer._generate_body_row(self, iter, max_cols, idx_lengths)
    814 if (r, c) in self.cell_context:
    815     cls = " " + self.cell_context[r, c]
    817 data_element = _element(
    818     "td",
    819     (
    820         f"{self.css['data']} {self.css['row']}{r} "
    821         f"{self.css['col']}{c}{cls}"
    822     ),
    823     value,
    824     data_element_visible,
    825     attributes="",
--> 826     display_value=self._display_funcs[(r, c)](value),
    827 )
    829 if self.cell_ids:
    830     data_element["id"] = f"{self.css['row']}{r}_{self.css['col']}{c}"

File [~/lincc/nested-pandas/src/nested_pandas/nestedframe/core.py:304](http://localhost:8888/~/lincc/nested-pandas/src/nested_pandas/nestedframe/core.py#line=303), in NestedFrame._repr_html_.<locals>.repack_row(chunk)
    303 def repack_row(chunk):
--> 304     return chunk.to_html(max_rows=1, max_cols=5, show_dimensions=True, index=False, header=False)

AttributeError: 'NoneType' object has no attribute 'to_html'
```

Should be a relatively simple fix to just check the type and adjust the output accordingly.

**Before submitting**
Please check the following:

- [x] I have described the situation in which the bug arose, including what code was executed, information about my environment, and any applicable data others will need to reproduce the problem.
- [x] I have included available evidence of the unexpected behavior (including error messages, screenshots, and/or plots) as well as a description of what I expected instead.
- [x] If I have a solution in mind, I have provided an explanation and/or pseudocode and/or task list.

Discussion:

Solution patch:
diff --git a/src/nested_pandas/nestedframe/core.py b/src/nested_pandas/nestedframe/core.py
index bdf2ce0c..ae7f3b81 100644
--- a/src/nested_pandas/nestedframe/core.py
+++ b/src/nested_pandas/nestedframe/core.py
@@ -291,6 +291,10 @@ def _repr_html_(self) -> str | None:
         # Nested Column Formatting
         # first cell shows the nested df header and a preview row
         def repack_first_cell(chunk):
+            # If the chunk is None, just return None
+            # Means that the column labels will not be shown
+            if chunk is None:
+                return None
             # Render header separately to keep data aligned
             output = chunk.head(0).to_html(
                 max_rows=0, max_cols=5, show_dimensions=False, index=False, header=True
@@ -301,6 +305,9 @@ def repack_first_cell(chunk):
 
         # remaining cells show only a preview row
         def repack_row(chunk):
+            # If the chunk is None, just return None
+            if chunk is None:
+                return None
             return chunk.to_html(max_rows=1, max_cols=5, show_dimensions=True, index=False, header=False)
 
         # Apply repacking to all nested columns
@@ -312,7 +319,9 @@ def repack_row(chunk):
         )
 
         # Recover some truncation formatting, limited to head truncation
-        if repr.data.shape[0] > pd.get_option("display.max_rows"):
+        if pd.get_option("display.max_rows") is None:
+            return repr.to_html(max_rows=0)
+        elif repr.data.shape[0] > pd.get_option("display.max_rows"):
             html_repr = repr.to_html(max_rows=pd.get_option("display.min_rows"))
         else:
             # when under the max_rows threshold, display all rows (behavior of 0 here)
@@ -916,6 +925,118 @@ def dropna(
             return None
         return new_df
 
+    def sort_values(
+        self,
+        by,
+        *,
+        axis=0,
+        ascending=True,
+        inplace=False,
+        kind="quicksort",
+        na_position="last",
+        ignore_index=False,
+        key=None,
+    ):
+        """
+        Sort by the values along either axis.
+
+        Parameters:
+        -----------
+        by : str or list of str
+            Name or list of names to sort by.
+
+            Access nested columns using `nested_df.nested_col` (where
+            `nested_df` refers to a particular nested dataframe and
+            `nested_col` is a column of that nested dataframe).
+        axis : {0 or 'index', 1 or 'columns'}, default 0
+            Axis to be sorted.
+        ascending : bool or list of bool, default True
+            Sort ascending vs. descending. Specify list for multiple sort
+            orders. If this is a list of bools, must match the length of the
+            by.
+        inplace : bool, default False
+            If True, perform operation in-place.
+        kind : {'quicksort', 'mergesort', 'heapsort'}, default 'quicksort'
+            Choice of sorting algorithm. See also ndarray.np.sort for more
+            information. mergesort is the only stable algorithm. For DataFrames,
+            this option is only applied when sorting on a single column or label.
+        na_position : {'first', 'last'}, default 'last'
+            Puts NaNs at the beginning if first; last puts NaNs at the end.
+        ignore_index : bool, default False
+            If True, the resulting axis will be labeled 0, 1, …, n - 1.
+            Always False when applied to nested layers.
+        key : callable, optional
+            Apply the key function to the values before sorting.
+
+        Returns:
+        --------
+        DataFrame or None
+            DataFrame with sorted values if inplace=False, None otherwise.
+        """
+
+        # Resolve target layer
+        target = []
+        if isinstance(by, str):
+            by = [by]
+        # Check "by" columns for hierarchical references
+        for col in by:
+            if self._is_known_hierarchical_column(col):
+                target.append(col.split(".")[0])
+            else:
+                target.append("base")
+
+        # Ensure one target layer, preventing multi-layer operations
+        target = np.unique(target)
+        if len(target) > 1:
+            raise ValueError("Queries cannot target multiple structs/layers, write a separate query for each")
+        target = str(target[0])
+
+        # Apply pandas sort_values
+        if target == "base":
+            return super().sort_values(
+                by=by,
+                axis=axis,
+                ascending=ascending,
+                inplace=inplace,
+                kind=kind,
+                na_position=na_position,
+                ignore_index=ignore_index,
+                key=key,
+            )
+        else:  # target is a nested column
+            target_flat = self[target].nest.to_flat()
+            target_flat = target_flat.set_index(self[target].array.get_list_index())
+
+            if target_flat.index.name is None:  # set name if not present
+                target_flat.index.name = "index"
+            # Index must always be the first sort key for nested columns
+            nested_by = [target_flat.index.name] + [col.split(".")[-1] for col in by]
+
+            # Augment the ascending kwarg to include the index
+            if isinstance(ascending, bool):
+                ascending = [True] + [ascending] * len(by)
+            elif isinstance(ascending, list):
+                ascending = [True] + ascending
+
+            target_flat = target_flat.sort_values(
+                by=nested_by,
+                axis=axis,
+                ascending=ascending,
+                kind=kind,
+                na_position=na_position,
+                ignore_index=False,
+                key=key,
+                inplace=False,
+            )
+
+            #  Could be optimized, as number of rows doesn't change
+            new_df = self._set_filtered_flat_df(nest_name=target, flat_df=target_flat)
+
+            if inplace:
+                self._update_inplace(new_df)
+                return None
+            return new_df
+
     def reduce(self, func, *args, **kwargs) -> NestedFrame:  # type: ignore[override]
         """
         Takes a function and applies it to each top-level row of the NestedFrame.
diff --git a/src/nested_pandas/series/ext_array.py b/src/nested_pandas/series/ext_array.py
index be610192..c1d152bd 100644
--- a/src/nested_pandas/series/ext_array.py
+++ b/src/nested_pandas/series/ext_array.py
@@ -51,7 +51,14 @@
 from pandas.io.formats.format import format_array
 
 from nested_pandas.series.dtype import NestedDtype
-from nested_pandas.series.utils import enumerate_chunks, is_pa_type_a_list
+from nested_pandas.series.utils import (
+    enumerate_chunks,
+    is_pa_type_a_list,
+    transpose_list_struct_array,
+    transpose_struct_list_array,
+    transpose_struct_list_type,
+    validate_struct_list_array_for_equal_lengths,
+)
 
 __all__ = ["NestedExtensionArray"]
 
@@ -549,6 +556,8 @@ def __arrow_array__(self, type=None):
         """Convert the extension array to a PyArrow array."""
         if type is None:
             return self._chunked_array
+        if isinstance(type, pa.ListType):
+            return self._list_array.cast(type)
         return self._chunked_array.cast(type)
 
     def __array__(self, dtype=None):
@@ -650,12 +659,27 @@ def __init__(self, values: pa.Array | pa.ChunkedArray, *, validate: bool = True)
         if isinstance(values, pa.Array):
             values = pa.chunked_array([values])
 
-        if validate:
+        # Convert list-struct array to struct-list array
+        if is_pa_type_a_list(values.type):
+            struct_chunks = []
+            for list_chunk in values.iterchunks():
+                struct_chunks.append(transpose_list_struct_array(list_chunk))
+            values = pa.chunked_array(struct_chunks)
+        # Validate struct-array with list fields
+        elif validate:
             self._validate(values)
 
         self._chunked_array = values
         self._dtype = NestedDtype(values.type)
 
+    @property
+    def _list_array(self) -> pa.ChunkedArray:
+        """Pyarrow chunked list-struct array representation"""
+        list_chunks = []
+        for struct_chunk in self._chunked_array.iterchunks():
+            list_chunks.append(transpose_struct_list_array(struct_chunk, validate=False))
+        return pa.chunked_array(list_chunks)
+
     @classmethod
     def from_sequence(cls, scalars, *, dtype: NestedDtype | pd.ArrowDtype | pa.DataType = None) -> Self:  # type: ignore[name-defined] # noqa: F821
         """Construct a NestedExtensionArray from a sequence of items
@@ -677,53 +701,65 @@ def from_sequence(cls, scalars, *, dtype: NestedDtype | pd.ArrowDtype | pa.DataT
         return cls._from_sequence(scalars, dtype=dtype)
 
     @property
-    def _pyarrow_dtype(self) -> pa.DataType:
+    def _pyarrow_dtype(self) -> pa.StructType:
         """PyArrow data type of the extension array"""
         return self._dtype.pyarrow_dtype
 
+    @property
+    def _pyarrow_list_struct_dtype(self) -> pa.ListType:
+        """PyArrow data type of the list-struct view over the ext. array"""
+        return transpose_struct_list_type(self._pyarrow_dtype)
+
     @property
     def chunked_array(self) -> pa.ChunkedArray:
         """The underlying PyArrow ChunkedArray"""
         return self._chunked_array
 
+    @property
+    def chunked_list_struct_array(self) -> pa.ChunkedArray:
+        """Chunked list-struct view over the extension array"""
+        return self._list_array
+
     @staticmethod
     def _validate(array: pa.ChunkedArray) -> None:
         """Raises ValueError if the input array is not a struct array with all fields being
         list arrays of the same lengths.
+
+        Parameters
+        ----------
+        array : pa.ChunkedArray
+            The array to validate.
+
+        Raises
+        ------
+        ValueError
         """
         for chunk in array.iterchunks():
-            if not pa.types.is_struct(chunk.type):
-                raise ValueError(f"Expected a StructArray, got {chunk.type}")
-            struct_array = cast(pa.StructArray, chunk)
-
-            first_list_array: pa.ListArray | None = None
-            for field in struct_array.type:
-                inner_array = struct_array.field(field.name)
-                if not is_pa_type_a_list(inner_array.type):
-                    raise ValueError(f"Expected a ListArray, got {inner_array.type}")
-                list_array = cast(pa.ListArray, inner_array)
-
-                if first_list_array is None:
-                    first_list_array = list_array
-                    continue
-                # compare offsets from the first list array with the current one
-                if not first_list_array.offsets.equals(list_array.offsets):
-                    raise ValueError("Offsets of all ListArrays must be the same")
+            validate_struct_list_array_for_equal_lengths(chunk)
 
     @classmethod
     def from_arrow_ext_array(cls, array: ArrowExtensionArray) -> Self:  # type: ignore[name-defined] # noqa: F821
         """Create a NestedExtensionArray from pandas' ArrowExtensionArray"""
         return cls(array._pa_array)
 
-    def to_arrow_ext_array(self) -> ArrowExtensionArray:
-        """Convert the extension array to pandas' ArrowExtensionArray"""
+    def to_arrow_ext_array(self, list_struct: bool = False) -> ArrowExtensionArray:
+        """Convert the extension array to pandas' ArrowExtensionArray
+
+        Parameters
+        ----------
+        list_struct : bool, optional
+            If False (default), return struct-list array, otherwise return
+            list-struct array.
+        """
+        if list_struct:
+            return ArrowExtensionArray(self._list_array)
         return ArrowExtensionArray(self._chunked_array)
 
     def _replace_chunked_array(self, pa_array: pa.ChunkedArray, *, validate: bool) -> None:
         if validate:
             self._validate(pa_array)
         self._chunked_array = pa_array
-        self._dtype = NestedDtype(pa_array.chunk(0).type)
+        self._dtype = NestedDtype(pa_array.type)
 
     @property
     def list_offsets(self) -> pa.Array:
@@ -737,48 +773,32 @@ def list_offsets(self) -> pa.Array:
         pa.ChunkedArray
             The list offsets of the field arrays.
         """
-        # Quick and cheap path for a single chunk
+        # Cheap path for a single chunk
         if self._chunked_array.num_chunks == 1:
             struct_array = cast(pa.StructArray, self._chunked_array.chunk(0))
             return cast(pa.ListArray, struct_array.field(0)).offsets
 
-        chunks = []
-        # The offset of the current chunk in the flat array.
-        # Offset arrays use int32 type, so we cast to it
-        chunk_offset = pa.scalar(0, type=pa.int32())
-        for chunk in self._chunked_array.iterchunks():
-            list_array = cast(pa.ListArray, chunk.field(0))
-            if chunk_offset.equals(pa.scalar(0, type=pa.int32())):
-                offsets = list_array.offsets
-            else:
-                offsets = pa.compute.add(list_array.offsets[1:], chunk_offset)
-            chunks.append(offsets)
-            chunk_offset = offsets[-1]
-        return pa.concat_arrays(chunks)
+        zero_and_lengths = pa.chunked_array(
+            [pa.array([0], type=pa.int32()), pa.array(self.list_lengths, type=pa.int32())]
+        )
+        offsets = pa.compute.cumulative_sum(zero_and_lengths)
+        return offsets.chunk(0) if offsets.num_chunks == 1 else offsets.combine_chunks()
 
     @property
     def field_names(self) -> list[str]:
         """Names of the nested columns"""
         return [field.name for field in self._chunked_array.chunk(0).type]
 
-    def _iter_list_lengths(self) -> Generator[int, None, None]:
-        """Iterate over the lengths of the list arrays"""
-        for chunk in self._chunked_array.iterchunks():
-            for length in chunk.field(0).value_lengths():
-                if length.is_valid:
-                    yield length.as_py()
-                else:
-                    yield 0
-
     @property
-    def list_lengths(self) -> list[int]:
+    def list_lengths(self) -> np.ndarray:
         """Lengths of the list arrays"""
-        return list(self._iter_list_lengths())
+        list_lengths = pa.compute.list_value_length(self._list_array)
+        return np.asarray(list_lengths)
 
     @property
     def flat_length(self) -> int:
         """Length of the flat arrays"""
-        return sum(self._iter_list_lengths())
+        return pa.compute.sum(self.list_lengths).as_py()
 
     @property
     def num_chunks(self) -> int:
@@ -790,8 +810,8 @@ def get_list_index(self) -> np.ndarray:
         if len(self) == 0:
             # Since we have no list offsets, return an empty array
             return np.array([], dtype=int)
-        list_index = np.arange(len(self))
-        return np.repeat(list_index, np.diff(self.list_offsets))
+        list_index = np.arange(len(self), dtype=int)
+        return np.repeat(list_index, self.list_lengths)
 
     def iter_field_lists(self, field: str) -> Generator[np.ndarray, None, None]:
         """Iterate over single field nested lists, as numpy arrays
@@ -813,7 +833,7 @@ def iter_field_lists(self, field: str) -> Generator[np.ndarray, None, None]:
                 yield np.asarray(list_scalar.values)
 
     def view_fields(self, fields: str | list[str]) -> Self:  # type: ignore[name-defined] # noqa: F821
-        """Get a view of the extension array with only the specified fields
+        """Get a view of the extension array with the specified fields only
 
         Parameters
         ----------
@@ -842,7 +862,7 @@ def view_fields(self, fields: str | list[str]) -> Self:  # type: ignore[name-def
             chunks.append(struct_array)
         pa_array = pa.chunked_array(chunks)
 
-        return self.__class__(pa_array, validate=False)
+        return type(self)(pa_array, validate=False)
 
     def set_flat_field(self, field: str, value: ArrayLike, *, keep_dtype: bool = False) -> None:
         """Set the field from flat-array of values
diff --git a/src/nested_pandas/series/utils.py b/src/nested_pandas/series/utils.py
index 34707c8b..9968ba61 100644
--- a/src/nested_pandas/series/utils.py
+++ b/src/nested_pandas/series/utils.py
@@ -1,9 +1,12 @@
+from __future__ import annotations  # Python 3.9 requires it for X | Y type hints
+
 from collections.abc import Generator
+from typing import cast
 
 import pyarrow as pa
 
 
-def is_pa_type_a_list(pa_type: type[pa.Array]) -> bool:
+def is_pa_type_a_list(pa_type: pa.DataType) -> bool:
     """Check if the given pyarrow type is a list type.
 
     I.e. one of the following types: ListArray, LargeListArray,
@@ -39,3 +42,148 @@ def enumerate_chunks(array: pa.ChunkedArray) -> Generator[tuple[slice, pa.Array]
         index_stop = index_start + len(chunk)
         yield slice(index_start, index_stop), chunk
         index_start = index_stop
+
+
+def validate_struct_list_array_for_equal_lengths(array: pa.StructArray) -> None:
+    """Check if the given struct array has lists of equal length.
+
+    Parameters
+    ----------
+    array : pa.StructArray
+        Input struct array.
+
+    Raises
+    ------
+    ValueError
+        If the struct array has lists of unequal length or type of the input
+        array is not a StructArray or fields are not ListArrays.
+    """
+    if not pa.types.is_struct(array.type):
+        raise ValueError(f"Expected a StructArray, got {array.type}")
+
+    first_list_array: pa.ListArray | None = None
+    for field in array.type:
+        inner_array = array.field(field.name)
+        if not is_pa_type_a_list(inner_array.type):
+            raise ValueError(f"Expected a ListArray, got {inner_array.type}")
+        list_array = cast(pa.ListArray, inner_array)
+
+        if first_list_array is None:
+            first_list_array = list_array
+            continue
+        # compare offsets from the first list array with the current one
+        if not first_list_array.offsets.equals(list_array.offsets):
+            raise ValueError("Offsets of all ListArrays must be the same")
+
+
+def transpose_struct_list_type(t: pa.StructType) -> pa.ListType:
+    """Converts a type of struct-list array into a type of list-struct array.
+
+    Parameters
+    ----------
+    t : pa.DataType
+        Input type of struct-list array.
+
+    Returns
+    -------
+    pa.DataType
+        Type of list-struct array.
+
+    Raises
+    ------
+    ValueError
+        If the input type is not a struct-list type.
+    """
+    if not pa.types.is_struct(t):
+        raise ValueError(f"Expected a StructType, got {t}")
+
+    fields = []
+    for field in t:
+        if not is_pa_type_a_list(field.type):
+            raise ValueError(f"Expected a ListType, got {field.type}")
+        list_type = cast(pa.ListType, field.type)
+        fields.append(pa.field(field.name, list_type.value_type))
+
+    list_type = cast(pa.ListType, pa.list_(pa.struct(fields)))
+    return list_type
+
+
+def transpose_struct_list_array(array: pa.StructArray, validate: bool = True) -> pa.ListArray:
+    """Converts a struct-array of lists into a list-array of structs.
+
+    Parameters
+    ----------
+    array : pa.StructArray
+        Input struct array, each scalar must have lists of equal length.
+    validate : bool, default True
+        Whether to validate the input array for list lengths. Raises ValueError
+        if something is wrong.
+
+    Returns
+    -------
+    pa.ListArray
+        List array of structs.
+    """
+    if validate:
+        validate_struct_list_array_for_equal_lengths(array)
+
+    # Since we know that all lists have the same length, we can use the first list to get offsets
+    offsets = array.field(0).offsets
+    struct_flat_array = pa.StructArray.from_arrays(
+        [field.values for field in array.flatten()],
+        names=array.type.names,
+    )
+    return pa.ListArray.from_arrays(offsets, struct_flat_array)
+
+
+def transpose_list_struct_type(t: pa.ListType) -> pa.StructType:
+    """Converts a type of list-struct array into a type of struct-list array.
+
+    Parameters
+    ----------
+    t : pa.DataType
+        Input type of list-struct array.
+
+    Returns
+    -------
+    pa.DataType
+        Type of struct-list array.
+
+    Raises
+    ------
+    ValueError
+        If the input type is not a list-struct type.
+    """
+    if not is_pa_type_a_list(t):
+        raise ValueError(f"Expected a ListType, got {t}")
+
+    struct_type = cast(pa.StructType, t.value_type)
+    fields = []
+    for field in struct_type:
+        fields.append(pa.field(field.name, pa.list_(field.type)))
+
+    struct_type = cast(pa.StructType, pa.struct(fields))
+    return struct_type
+
+
+def transpose_list_struct_array(array: pa.ListArray) -> pa.StructArray:
+    """Converts a list-array of structs into a struct-array of lists.
+
+    Parameters
+    ----------
+    array : pa.ListArray
+        Input list array of structs.
+
+    Returns
+    -------
+    pa.StructArray
+        Struct array of lists.
+    """
+    offsets, values = array.offsets, array.values
+
+    fields = []
+    for field_values in values.flatten():
+        list_array = pa.ListArray.from_arrays(offsets, field_values)
+        fields.append(list_array)
+
+    return pa.StructArray.from_arrays(fields, names=array.type.value_type.names)


Test patch:
diff --git a/tests/nested_pandas/nestedframe/test_nestedframe.py b/tests/nested_pandas/nestedframe/test_nestedframe.py
index a582db7b..80b8e184 100644
--- a/tests/nested_pandas/nestedframe/test_nestedframe.py
+++ b/tests/nested_pandas/nestedframe/test_nestedframe.py
@@ -807,6 +807,60 @@ def test_dropna_errors():
         base.dropna(on_nested="nested", subset=["b"])
 
 
+def test_sort_values():
+    """Test that sort_values works on all layers"""
+
+    base = NestedFrame(data={"a": [1, 2, 3], "b": [2, 3, 6]}, index=[0, 1, 2])
+
+    nested = pd.DataFrame(
+        data={"c": [0, 2, 4, 1, 4, 3, 1, 4, 1], "d": [5, 4, 7, 5, 3, 1, 9, 3, 4]},
+        index=[0, 0, 0, 1, 1, 1, 2, 2, 2],
+    )
+
+    base = base.add_nested(nested, "nested")
+
+    # Test basic functionality
+    sv_base = base.sort_values("b")
+    assert list(sv_base.index) == [0, 1, 2]
+
+    # Test on nested column
+    sv_base = base.sort_values(["nested.d"])
+    assert list(sv_base.iloc[0]["nested"]["d"]) == [4, 5, 7]
+
+    # Test multi-layer error trigger
+    with pytest.raises(ValueError):
+        base.sort_values(["a", "nested.c"])
+
+    # Test inplace=True
+    base.sort_values("nested.d", inplace=True)
+    assert list(base.iloc[0]["nested"]["d"]) == [4, 5, 7]
+
+
+def test_sort_values_ascension():
+    """Test that sort_values works with various ascending settings"""
+
+    base = NestedFrame(data={"a": [1, 2, 3], "b": [2, 3, 6]}, index=[0, 1, 2])
+
+    nested = pd.DataFrame(
+        data={"c": [0, 2, 4, 1, 4, 3, 1, 4, 1], "d": [5, 4, 7, 5, 3, 1, 9, 3, 4]},
+        index=[0, 0, 0, 1, 1, 1, 2, 2, 2],
+    )
+
+    base = base.add_nested(nested, "nested")
+
+    # Test ascending=False
+    sv_base = base.sort_values("nested.d", ascending=False)
+    assert list(sv_base.iloc[0]["nested"]["d"]) == [7, 5, 4]
+
+    # Test list ascending
+    sv_base = base.sort_values("nested.d", ascending=[False])
+    assert list(sv_base.iloc[0]["nested"]["d"]) == [7, 5, 4]
+
+    # Test multi-by multi-ascending
+    sv_base = base.sort_values(["nested.d", "nested.c"], ascending=[False, True])
+    assert list(sv_base.iloc[0]["nested"]["d"]) == [7, 5, 4]
+
+
 def test_reduce():
     """Tests that we can call reduce on a NestedFrame with a custom function."""
     nf = NestedFrame(
diff --git a/tests/nested_pandas/series/test_accessor.py b/tests/nested_pandas/series/test_accessor.py
index 1f195ce2..4bc46e74 100644
--- a/tests/nested_pandas/series/test_accessor.py
+++ b/tests/nested_pandas/series/test_accessor.py
@@ -296,7 +296,7 @@ def test_list_lengths():
         ]
     )
     assert series.shape == (3,)
-    assert series.nest.list_lengths == [3, 0, 2]
+    assert_array_equal(series.nest.list_lengths, [3, 0, 2])
 
 
 def test_flat_length():
diff --git a/tests/nested_pandas/series/test_ext_array.py b/tests/nested_pandas/series/test_ext_array.py
index e580f902..98a7e063 100644
--- a/tests/nested_pandas/series/test_ext_array.py
+++ b/tests/nested_pandas/series/test_ext_array.py
@@ -603,6 +603,29 @@ def test_chunked_array():
     assert ext_array.chunked_array == pa.chunked_array(struct_array)
 
 
+def test_chunked_list_struct_array():
+    """Test that .chunked_list_struct_array is correct."""
+    struct_array = pa.StructArray.from_arrays(
+        arrays=[
+            pa.array([np.array([1, 2, 3]), np.array([1, 2, 1])]),
+            pa.array([-np.array([4.0, 5.0, 6.0]), -np.array([3.0, 4.0, 5.0])]),
+        ],
+        names=["a", "b"],
+    )
+    ext_array = NestedExtensionArray(struct_array)
+
+    list_array = pa.array(
+        [
+            [{"a": 1, "b": -4.0}, {"a": 2, "b": -5.0}, {"a": 3, "b": -6.0}],
+            [{"a": 1, "b": -3.0}, {"a": 2, "b": -4.0}, {"a": 1, "b": -5.0}],
+        ]
+    )
+    desired = pa.chunked_array([list_array])
+    # pyarrow returns a single bool for ==
+    assert ext_array.chunked_list_struct_array == desired
+    assert ext_array.chunked_list_struct_array.type == ext_array._pyarrow_list_struct_dtype
+
+
 def test_list_offsets_single_chunk():
     """Test that the .list_offset property is correct for a single chunk."""
     struct_array = pa.StructArray.from_arrays(
@@ -1044,6 +1067,26 @@ def test___arrow_array___with_type_cast():
     assert arrow_array == struct_array.cast(new_pa_type)
 
 
+def test___arrow___with_list_type():
+    """Test that the extension array can be converted to a pyarrow array with a list type."""
+    struct_array = pa.StructArray.from_arrays(
+        arrays=[
+            pa.array([np.array([1, 2, 3]), np.array([1, 2, 1])]),
+            pa.array([-np.array([4, 5, 6]), -np.array([3, 4, 5])]),
+        ],
+        names=["a", "b"],
+    )
+    ext_array = NestedExtensionArray(struct_array)
+    list_pa_type = pa.list_(pa.struct([pa.field("a", pa.int64()), pa.field("b", pa.int64())]))
+    desired_list_array = pa.array(
+        [
+            [{"a": 1, "b": -4}, {"a": 2, "b": -5}, {"a": 3, "b": -6}],
+            [{"a": 1, "b": -3}, {"a": 2, "b": -4}, {"a": 1, "b": -5}],
+        ]
+    )
+    assert pa.array(ext_array, type=list_pa_type) == desired_list_array
+
+
 def test___array__():
     """Test that the extensions array can be converted to a numpy array and back."""
     struct_array = pa.array(
@@ -1241,7 +1284,7 @@ def test_list_lengths():
         pa.chunked_array([struct_array, empty_struct_array, struct_array, null_struct_array])
     )
 
-    assert ext_array.list_lengths == [3, 4, 3, 4, 0]
+    assert_array_equal(ext_array.list_lengths, [3, 4, 3, 4, 0])
 
 
 def test_flat_length():
@@ -1738,6 +1781,25 @@ def test_to_arrow_ext_array():
     assert_series_equal(pd.Series(ext_array), pd.Series(to_arrow), check_dtype=False)
 
 
+def test_to_arrow_ext_array_with_list_struct_true():
+    """Tests that we are getting a valid ArrowExtensionArray when list_struct=True."""
+    struct_array = pa.StructArray.from_arrays(
+        arrays=[
+            pa.array([np.array([1, 2, 3]), np.array([1, 2, 1, 2])]),
+            pa.array([-np.array([4.0, 5.0, 6.0]), -np.array([3.0, 4.0, 5.0, 6.0])]),
+        ],
+        names=["a", "b"],
+    )
+    nested_ext_array = NestedExtensionArray(struct_array)
+
+    arrow_ext_array = nested_ext_array.to_arrow_ext_array(list_struct=True)
+    assert_frame_equal(
+        pd.Series(arrow_ext_array).list.flatten().struct.explode(),
+        pd.Series(nested_ext_array).nest.to_flat().reset_index(drop=True),
+        check_index_type=False,
+    )
+
+
 def test_series_interpolate():
     """We do not support interpolate() on NestedExtensionArray."""
     with pytest.raises(NotImplementedError):
@@ -1746,6 +1808,24 @@ def test_series_interpolate():
         ).interpolate()
 
 
+def test___init___with_list_struct_array():
+    """Check if construction from list-struct pyarrow array works"""
+    list_array = pa.array(
+        [[{"a": 1, "b": 2}, {"a": 3, "b": 4}], [{"a": 5, "b": 6}], [{"a": -1, "b": -2}], []]
+    )
+    ext_array = NestedExtensionArray(list_array)
+    assert ext_array.field_names == ["a", "b"]
+    assert ext_array.flat_length == 4
+    struct_array = pa.StructArray.from_arrays(
+        arrays=[
+            pa.array([[1, 3], [5], [-1], []]),
+            pa.array([[2, 4], [6], [-2], []]),
+        ],
+        names=["a", "b"],
+    )
+    assert pa.array(ext_array) == struct_array
+
+
 def test__from_sequence_of_strings():
     """We do not support from_sequence_of_strings() which would apply things like pd.read_csv()"""
     with pytest.raises(NotImplementedError):
diff --git a/tests/nested_pandas/series/test_series_utils.py b/tests/nested_pandas/series/test_series_utils.py
new file mode 100644
index 00000000..7fae7724
--- /dev/null
+++ b/tests/nested_pandas/series/test_series_utils.py
@@ -0,0 +1,113 @@
+import pyarrow as pa
+import pytest
+from nested_pandas.series.utils import (
+    transpose_list_struct_array,
+    transpose_list_struct_type,
+    transpose_struct_list_array,
+    transpose_struct_list_type,
+    validate_struct_list_array_for_equal_lengths,
+)
+
+
+def test_validate_struct_list_array_for_equal_lengths():
+    """Test validate_struct_list_array_for_equal_lengths function."""
+    # Raises for wrong types
+    with pytest.raises(ValueError):
+        validate_struct_list_array_for_equal_lengths(pa.array([], type=pa.int64()))
+    with pytest.raises(ValueError):
+        validate_struct_list_array_for_equal_lengths(pa.array([], type=pa.list_(pa.int64())))
+
+    # Raises if one of the fields is not a ListArray
+    with pytest.raises(ValueError):
+        validate_struct_list_array_for_equal_lengths(
+            pa.StructArray.from_arrays([pa.array([[1, 2], [3, 4, 5]]), pa.array([1, 2])], ["a", "b"])
+        )
+
+    # Raises for mismatched lengths
+    with pytest.raises(ValueError):
+        validate_struct_list_array_for_equal_lengths(
+            pa.StructArray.from_arrays(
+                [pa.array([[1, 2], [3, 4, 5]]), pa.array([[1, 2, 3], [4, 5]])], ["a", "b"]
+            )
+        )
+
+    input_array = pa.StructArray.from_arrays(
+        arrays=[
+            pa.array([[1, 2], [3, 4], [], [5, 6, 7]]),
+            pa.array([["x", "y"], ["y", "x"], [], ["d", "e", "f"]]),
+        ],
+        names=["a", "b"],
+    )
+    assert validate_struct_list_array_for_equal_lengths(input_array) is None
+
+
+def test_transpose_struct_list_type():
+    """Test transpose_struct_list_type function."""
+    # Raises for wrong types
+    with pytest.raises(ValueError):
+        transpose_struct_list_type(pa.int64())
+    with pytest.raises(ValueError):
+        transpose_struct_list_type(pa.list_(pa.int64()))
+
+    # Raises if one of the fields is not a ListType
+    with pytest.raises(ValueError):
+        transpose_struct_list_type(pa.struct([("a", pa.int64()), ("b", pa.int64())]))
+
+    input_type = pa.struct([("a", pa.list_(pa.int64())), ("b", pa.list_(pa.string()))])
+    expected_output = pa.list_(pa.struct([("a", pa.int64()), ("b", pa.string())]))
+    assert transpose_struct_list_type(input_type) == expected_output
+
+
+def test_transpose_list_struct_type():
+    """Test transpose_list_struct_type function."""
+    # Raises for wrong types
+    with pytest.raises(ValueError):
+        transpose_list_struct_type(pa.int64())
+    with pytest.raises(ValueError):
+        transpose_list_struct_type(pa.struct([("a", pa.int64()), ("b", pa.int64())]))
+
+    input_type = pa.list_(pa.struct([("a", pa.int64()), ("b", pa.string())]))
+    expected_output = pa.struct([("a", pa.list_(pa.int64())), ("b", pa.list_(pa.string()))])
+    assert transpose_list_struct_type(input_type) == expected_output
+
+
+def test_transpose_struct_list_array():
+    """Test transpose_struct_list_array function."""
+    input_array = pa.StructArray.from_arrays(
+        arrays=[
+            pa.array([[1, 2], [3, 4], [], [5, 6, 7]]),
+            pa.array([["x", "y"], ["y", "x"], [], ["d", "e", "f"]]),
+        ],
+        names=["a", "b"],
+    )
+    desired = pa.array(
+        [
+            [{"a": 1, "b": "x"}, {"a": 2, "b": "y"}],
+            [{"a": 3, "b": "y"}, {"a": 4, "b": "x"}],
+            [],
+            [{"a": 5, "b": "d"}, {"a": 6, "b": "e"}, {"a": 7, "b": "f"}],
+        ]
+    )
+    actual = transpose_struct_list_array(input_array)
+    assert actual == desired
+
+
+def test_transpose_list_struct_array():
+    """Test transpose_list_struct_array function."""
+    input_array = pa.array(
+        [
+            [{"a": 1, "b": "x"}, {"a": 2, "b": "y"}],
+            [{"a": 3, "b": "y"}, {"a": 4, "b": "x"}],
+            [],
+            [{"a": 5, "b": "d"}, {"a": 6, "b": "e"}, {"a": 7, "b": "f"}],
+        ]
+    )
+    desired = pa.StructArray.from_arrays(
+        arrays=[
+            pa.array([[1, 2], [3, 4], [], [5, 6, 7]]),
+            pa.array([["x", "y"], ["y", "x"], [], ["d", "e", "f"]]),
+        ],
+        names=["a", "b"],
+    )
+    actual = transpose_list_struct_array(input_array)
+    assert actual == desired