Repository: modin-project/modin

Issue:
Modin DataFrame filter functionality (internal)
`query` and `dropna` could benefit from a filter-specific to modifying a single axis. At the moment, they are using `_apply...` which does not guarantee or cannot infer metadata information related to the sizes of partitions. We could improve the performance by maintaining the partition sizes along the opposite axis from which we are filtering.


Discussion:

Resolving patch:
diff --git a/modin/backends/pandas/query_compiler.py b/modin/backends/pandas/query_compiler.py
--- a/modin/backends/pandas/query_compiler.py
+++ b/modin/backends/pandas/query_compiler.py
@@ -969,9 +969,7 @@ def query_builder(df, **kwargs):
             return df.query(expr, inplace=False, **kwargs)
 
         return self.__constructor__(
-            self._modin_frame._apply_full_axis(
-                1, query_builder, new_columns=self.columns
-            )
+            self._modin_frame.filter_full_axis(1, query_builder)
         )
 
     def rank(self, **kwargs):
@@ -1162,67 +1160,12 @@ def dropna(self, **kwargs):
         Return:
             a new QueryCompiler
         """
-        axis = kwargs.get("axis", 0)
-        subset = kwargs.get("subset", None)
-        thresh = kwargs.get("thresh", None)
-        how = kwargs.get("how", "any")
-        # We need to subset the axis that we care about with `subset`. This
-        # will be used to determine the number of values that are NA.
-        if subset is not None:
-            if not axis:
-                compute_na = self.getitem_column_array(subset)
-            else:
-                compute_na = self.getitem_row_array(self.index.get_indexer_for(subset))
-        else:
-            compute_na = self
-
-        if not isinstance(axis, list):
-            axis = [axis]
-        # We are building this dictionary first to determine which columns
-        # and rows to drop. This way we do not drop some columns before we
-        # know which rows need to be dropped.
-        if thresh is not None:
-            # Count the number of NA values and specify which are higher than
-            # thresh.
-            drop_values = {
-                ax ^ 1: compute_na.isna().sum(axis=ax ^ 1).to_pandas().squeeze()
-                > thresh
-                for ax in axis
-            }
-        else:
-            drop_values = {
-                ax
-                ^ 1: getattr(compute_na.isna(), how)(axis=ax ^ 1).to_pandas().squeeze()
-                for ax in axis
-            }
-
-        if 0 not in drop_values:
-            drop_values[0] = None
-
-        if 1 not in drop_values:
-            drop_values[1] = None
-
-            rm_from_index = (
-                [obj for obj in compute_na.index[drop_values[1]]]
-                if drop_values[1] is not None
-                else None
-            )
-            rm_from_columns = (
-                [obj for obj in compute_na.columns[drop_values[0]]]
-                if drop_values[0] is not None
-                else None
-            )
-        else:
-            rm_from_index = (
-                compute_na.index[drop_values[1]] if drop_values[1] is not None else None
-            )
-            rm_from_columns = (
-                compute_na.columns[drop_values[0]]
-                if drop_values[0] is not None
-                else None
+        return self.__constructor__(
+            self._modin_frame.filter_full_axis(
+                kwargs.get("axis", 0) ^ 1,
+                lambda df: pandas.DataFrame.dropna(df, **kwargs),
             )
-
-        return self.drop(index=rm_from_index, columns=rm_from_columns)
+        )
 
     def drop(self, index=None, columns=None):
         """Remove row data for target index and columns.
diff --git a/modin/engines/base/frame/data.py b/modin/engines/base/frame/data.py
--- a/modin/engines/base/frame/data.py
+++ b/modin/engines/base/frame/data.py
@@ -780,6 +780,43 @@ def _fold(self, axis, func):
             self._column_widths,
         )
 
+    def filter_full_axis(self, axis, func):
+        """Filter data based on the function provided along an entire axis.
+
+        Args:
+            axis: The axis to filter over.
+            func: The function to use for the filter. This function should filter the
+                data itself.
+
+        Returns:
+            A new dataframe.
+        """
+        new_partitions = self._frame_mgr_cls.map_axis_partitions(
+            axis, self._partitions, func
+        )
+        if axis == 0:
+            new_index = self.index
+            new_lengths = self._row_lengths
+            new_widths = None  # We do not know what the resulting widths will be
+            new_columns = self._frame_mgr_cls.get_indices(
+                1, new_partitions, lambda df: df.columns
+            )
+        else:
+            new_columns = self.columns
+            new_lengths = None  # We do not know what the resulting lengths will be
+            new_widths = self._column_widths
+            new_index = self._frame_mgr_cls.get_indices(
+                0, new_partitions, lambda df: df.index
+            )
+        return self.__constructor__(
+            new_partitions,
+            new_index,
+            new_columns,
+            new_lengths,
+            new_widths,
+            self.dtypes if axis == 0 else None,
+        )
+
     def _apply_full_axis(
         self, axis, func, new_index=None, new_columns=None, dtypes=None
     ):


Test patch: