Repository: modin-project/modin

Issue:
Subclassing modin.pandas Dataframe

Apologies if this follow under bug but I wasn't sure. I was trying to subclass modin to add on additional methods for our needs but I noticed that the subclassing works for the first result however when the dataframe is filtered or when a new dataframe is returned we always get a instance of modin dataframe rather than our own dataframe that we extended from modin.pandas dataframe. 

I did see traces of code where the return is `type(self)(...)` which would return our extended dataframe but make place the return is `Dataframe(...)` explicitly.


Discussion:
Hi @denish44, thanks for posting!

I definitely agree, ideally we will always use the `__constructor__` pattern we created, looking through the code there are some places that haven't been updated in a while that still use `DataFrame(...)`. For example:

https://github.com/modin-project/modin/blob/6550939753c76e896ef2bfd65bb9468d6ad161d7/modin/pandas/dataframe.py#L336

I agree, we should make these changes to ensure that the DataFrame can be subclassed. If you are able, and want to make the subset of changes necessary for your usecase I would be happy to accept a PR that solves part of the issue in the interest of unblocking you. 
Not blocked by this but it is good first task to contribute. I will work on getting a PR ready for this, since the pattern is already setup don't think it will be too bad of change. It might be a few days.
@denish44 do you need any help from us to get a PR to fix this?

Resolving patch:
diff --git a/modin/pandas/dataframe.py b/modin/pandas/dataframe.py
--- a/modin/pandas/dataframe.py
+++ b/modin/pandas/dataframe.py
@@ -142,7 +142,7 @@ def __init__(
                 # the DataFrame and sets columns to the columns provided.
                 elif columns is not None and data.name not in columns:
                     self._query_compiler = from_pandas(
-                        DataFrame(columns=columns)
+                        self.__constructor__(columns=columns)
                     )._query_compiler
                 if index is not None:
                     self._query_compiler = data.loc[index]._query_compiler
@@ -373,18 +373,22 @@ def add_prefix(self, prefix):  # noqa: PR01, RT01, D200
         """
         Prefix labels with string `prefix`.
         """
-        return DataFrame(query_compiler=self._query_compiler.add_prefix(prefix))
+        return self.__constructor__(
+            query_compiler=self._query_compiler.add_prefix(prefix)
+        )
 
     def add_suffix(self, suffix):  # noqa: PR01, RT01, D200
         """
         Suffix labels with string `suffix`.
         """
-        return DataFrame(query_compiler=self._query_compiler.add_suffix(suffix))
+        return self.__constructor__(
+            query_compiler=self._query_compiler.add_suffix(suffix)
+        )
 
     def applymap(self, func, na_action: Optional[str] = None, **kwargs):
         if not callable(func):
             raise ValueError("'{0}' object is not callable".format(type(func)))
-        return DataFrame(
+        return self.__constructor__(
             query_compiler=self._query_compiler.applymap(
                 func, na_action=na_action, **kwargs
             )
@@ -562,7 +566,9 @@ def transpose(self, copy=False, *args):  # noqa: PR01, RT01, D200
         """
         # FIXME: Judging by pandas docs `*args` serves only compatibility purpose
         # and does not affect the result, we shouldn't pass it to the query compiler.
-        return DataFrame(query_compiler=self._query_compiler.transpose(*args))
+        return self.__constructor__(
+            query_compiler=self._query_compiler.transpose(*args)
+        )
 
     T = property(transpose)
 
@@ -613,7 +619,7 @@ def append(
                 other = other._query_compiler.transpose()
         elif isinstance(other, list):
             if not all(isinstance(o, BasePandasDataset) for o in other):
-                other = DataFrame(pandas.DataFrame(other))._query_compiler
+                other = self.__constructor__(pandas.DataFrame(other))._query_compiler
             else:
                 other = [obj._query_compiler for obj in other]
         else:
@@ -638,7 +644,7 @@ def append(
         query_compiler = self._query_compiler.concat(
             0, other, ignore_index=ignore_index, sort=sort
         )
-        return DataFrame(query_compiler=query_compiler)
+        return self.__constructor__(query_compiler=query_compiler)
 
     def assign(self, **kwargs):  # noqa: PR01, RT01, D200
         """
@@ -853,7 +859,7 @@ def equals(self, other):  # noqa: PR01, RT01, D200
         """
         if isinstance(other, pandas.DataFrame):
             # Copy into a Modin DataFrame to simplify logic below
-            other = DataFrame(other)
+            other = self.__constructor__(other)
         return (
             self.index.equals(other.index)
             and self.columns.equals(other.columns)
@@ -1218,11 +1224,11 @@ def insert(
                     )
             new_index = value.index.copy()
             new_columns = self.columns.insert(loc, column)
-            new_query_compiler = DataFrame(
+            new_query_compiler = self.__constructor__(
                 value, index=new_index, columns=new_columns
             )._query_compiler
         elif len(self.columns) == 0 and loc == 0:
-            new_query_compiler = DataFrame(
+            new_query_compiler = self.__constructor__(
                 data=value, columns=[column], index=self.index
             )._query_compiler
         else:
@@ -1349,7 +1355,7 @@ def join(
         if isinstance(other, Series):
             if other.name is None:
                 raise ValueError("Other Series must have a name")
-            other = DataFrame({other.name: other})
+            other = self.__constructor__({other.name: other})
         if on is not None:
             return self.__constructor__(
                 query_compiler=self._query_compiler.join(
@@ -1386,7 +1392,7 @@ def join(
                 )
                 .columns
             )
-        new_frame = DataFrame(
+        new_frame = self.__constructor__(
             query_compiler=self._query_compiler.concat(
                 1, [obj._query_compiler for obj in other], join=how, sort=sort
             )
@@ -1576,13 +1582,15 @@ def nlargest(self, n, columns, keep="first"):  # noqa: PR01, RT01, D200
         """
         Return the first `n` rows ordered by `columns` in descending order.
         """
-        return DataFrame(query_compiler=self._query_compiler.nlargest(n, columns, keep))
+        return self.__constructor__(
+            query_compiler=self._query_compiler.nlargest(n, columns, keep)
+        )
 
     def nsmallest(self, n, columns, keep="first"):  # noqa: PR01, RT01, D200
         """
         Return the first `n` rows ordered by `columns` in ascending order.
         """
-        return DataFrame(
+        return self.__constructor__(
             query_compiler=self._query_compiler.nsmallest(
                 n=n, columns=columns, keep=keep
             )
@@ -1597,7 +1605,7 @@ def slice_shift(self, periods=1, axis=0):  # noqa: PR01, RT01, D200
 
         if axis == "index" or axis == 0:
             if abs(periods) >= len(self.index):
-                return DataFrame(columns=self.columns)
+                return self.__constructor__(columns=self.columns)
             else:
                 new_df = self.iloc[:-periods] if periods > 0 else self.iloc[-periods:]
                 new_df.index = (
@@ -1606,7 +1614,7 @@ def slice_shift(self, periods=1, axis=0):  # noqa: PR01, RT01, D200
                 return new_df
         else:
             if abs(periods) >= len(self.columns):
-                return DataFrame(index=self.index)
+                return self.__constructor__(index=self.index)
             else:
                 new_df = (
                     self.iloc[:, :-periods] if periods > 0 else self.iloc[:, -periods:]
@@ -1629,7 +1637,7 @@ def unstack(self, level=-1, fill_value=None):  # noqa: PR01, RT01, D200
                 query_compiler=self._query_compiler.unstack(level, fill_value)
             )
         else:
-            return DataFrame(
+            return self.__constructor__(
                 query_compiler=self._query_compiler.unstack(level, fill_value)
             )
 
@@ -1659,7 +1667,7 @@ def pivot_table(
         """
         Create a spreadsheet-style pivot table as a ``DataFrame``.
         """
-        result = DataFrame(
+        result = self.__constructor__(
             query_compiler=self._query_compiler.pivot_table(
                 index=index,
                 values=values,
@@ -2153,7 +2161,9 @@ def stack(self, level=-1, dropna=True):  # noqa: PR01, RT01, D200
                 query_compiler=self._query_compiler.stack(level, dropna)
             )
         else:
-            return DataFrame(query_compiler=self._query_compiler.stack(level, dropna))
+            return self.__constructor__(
+                query_compiler=self._query_compiler.stack(level, dropna)
+            )
 
     def sub(
         self, other, axis="columns", level=None, fill_value=None
@@ -2487,7 +2497,7 @@ def update(
         Modify in place using non-NA values from another ``DataFrame``.
         """
         if not isinstance(other, DataFrame):
-            other = DataFrame(other)
+            other = self.__constructor__(other)
         query_compiler = self._query_compiler.df_update(
             other._query_compiler,
             join=join,
@@ -2536,7 +2546,7 @@ def where(
                 cond = np.asanyarray(cond)
             if cond.shape != self.shape:
                 raise ValueError("Array conditional must be same shape as self")
-            cond = DataFrame(cond, index=self.index, columns=self.columns)
+            cond = self.__constructor__(cond, index=self.index, columns=self.columns)
         if isinstance(other, DataFrame):
             other = other._query_compiler
         else:
@@ -2608,7 +2618,7 @@ def _getitem_column(self, key):
         """
         if key not in self.keys():
             raise KeyError("{}".format(key))
-        s = DataFrame(
+        s = self.__constructor__(
             query_compiler=self._query_compiler.getitem_column_array([key])
         ).squeeze(axis=1)
         if isinstance(s, Series):
@@ -2716,7 +2726,7 @@ def __setitem__(self, key, value):
                 if isinstance(key, np.ndarray):
                     if key.shape != self.shape:
                         raise ValueError("Array must be same shape as DataFrame")
-                    key = DataFrame(key, columns=self.columns)
+                    key = self.__constructor__(key, columns=self.columns)
                 return self.mask(key, value, inplace=True)
 
             if isinstance(key, list) and all((x in self.columns for x in key)):
@@ -2781,7 +2791,7 @@ def setitem_unhashable_key(df, value):
                 value = list(value)
 
         if not self._query_compiler.lazy_execution and len(self.index) == 0:
-            new_self = DataFrame({key: value}, columns=self.columns)
+            new_self = self.__constructor__({key: value}, columns=self.columns)
             self._update_inplace(new_self._query_compiler)
         else:
             if isinstance(value, Series):
@@ -2938,7 +2948,7 @@ def _create_or_update_from_compiler(self, new_query_compiler, inplace=False):
             or type(new_query_compiler) in self._query_compiler.__class__.__bases__
         ), "Invalid Query Compiler object: {}".format(type(new_query_compiler))
         if not inplace:
-            return DataFrame(query_compiler=new_query_compiler)
+            return self.__constructor__(query_compiler=new_query_compiler)
         else:
             self._update_inplace(new_query_compiler=new_query_compiler)
 
@@ -3197,11 +3207,13 @@ def _getitem(self, key):
         except (KeyError, ValueError, TypeError):
             pass
         if isinstance(key, Series):
-            return DataFrame(
+            return self.__constructor__(
                 query_compiler=self._query_compiler.getitem_array(key._query_compiler)
             )
         elif isinstance(key, (np.ndarray, pandas.Index, list)):
-            return DataFrame(query_compiler=self._query_compiler.getitem_array(key))
+            return self.__constructor__(
+                query_compiler=self._query_compiler.getitem_array(key)
+            )
         elif isinstance(key, DataFrame):
             return self.where(key)
         elif is_mi_columns:
diff --git a/modin/pandas/series.py b/modin/pandas/series.py
--- a/modin/pandas/series.py
+++ b/modin/pandas/series.py
@@ -515,13 +515,17 @@ def add_prefix(self, prefix):  # noqa: PR01, RT01, D200
         """
         Prefix labels with string `prefix`.
         """
-        return Series(query_compiler=self._query_compiler.add_prefix(prefix, axis=0))
+        return self.__constructor__(
+            query_compiler=self._query_compiler.add_prefix(prefix, axis=0)
+        )
 
     def add_suffix(self, suffix):  # noqa: PR01, RT01, D200
         """
         Suffix labels with string `suffix`.
         """
-        return Series(query_compiler=self._query_compiler.add_suffix(suffix, axis=0))
+        return self.__constructor__(
+            query_compiler=self._query_compiler.add_suffix(suffix, axis=0)
+        )
 
     def append(
         self, to_append, ignore_index=False, verify_integrity=False
@@ -597,7 +601,7 @@ def append(
         if len(query_compiler.columns) > 1:
             return DataFrame(query_compiler=query_compiler)
         else:
-            return Series(query_compiler=query_compiler)
+            return self.__constructor__(query_compiler=query_compiler)
 
     def aggregate(self, func=None, axis=0, *args, **kwargs):  # noqa: PR01, RT01, D200
         """
@@ -692,7 +696,7 @@ def f(x):
 
             result = DataFrame(query_compiler=result)
         elif return_type == "Series":
-            result = Series(query_compiler=result)
+            result = self.__constructor__(query_compiler=result)
             if result.name == self.index[0]:
                 result.name = None
         elif isinstance(result, type(self._query_compiler)):
@@ -1358,7 +1362,9 @@ def nsmallest(self, n=5, keep="first"):  # noqa: PR01, RT01, D200
         """
         Return the smallest `n` elements.
         """
-        return Series(query_compiler=self._query_compiler.nsmallest(n=n, keep=keep))
+        return self.__constructor__(
+            query_compiler=self._query_compiler.nsmallest(n=n, keep=keep)
+        )
 
     def slice_shift(self, periods=1, axis=0):  # noqa: PR01, RT01, D200
         """
@@ -1369,7 +1375,7 @@ def slice_shift(self, periods=1, axis=0):  # noqa: PR01, RT01, D200
 
         if axis == "index" or axis == 0:
             if abs(periods) >= len(self.index):
-                return Series(dtype=self.dtype)
+                return self.__constructor__(dtype=self.dtype)
             else:
                 new_df = self.iloc[:-periods] if periods > 0 else self.iloc[-periods:]
                 new_df.index = (
@@ -2031,7 +2037,7 @@ def update(self, other):  # noqa: PR01, D200
         Modify Series in place using values from passed Series.
         """
         if not isinstance(other, Series):
-            other = Series(other)
+            other = self.__constructor__(other)
         query_compiler = self._query_compiler.series_update(other._query_compiler)
         self._update_inplace(new_query_compiler=query_compiler)
 
@@ -2044,7 +2050,7 @@ def value_counts(
         if bins is not None:
             # Potentially we could implement `cut` function from pandas API, which
             # bins values into intervals, and then we can just count them as regular values.
-            # TODO #1333: new_self = Series(pd.cut(self, bins, include_lowest=True), dtype="interval")
+            # TODO #1333: new_self = self.__constructor__(pd.cut(self, bins, include_lowest=True), dtype="interval")
             return self._default_to_pandas(
                 pandas.Series.value_counts,
                 normalize=normalize,
@@ -2426,7 +2432,7 @@ def _create_or_update_from_compiler(self, new_query_compiler, inplace=False):
             or type(new_query_compiler) in self._query_compiler.__class__.__bases__
         ), "Invalid Query Compiler object: {}".format(type(new_query_compiler))
         if not inplace and new_query_compiler.is_series_like():
-            return Series(query_compiler=new_query_compiler)
+            return self.__constructor__(query_compiler=new_query_compiler)
         elif not inplace:
             # This can happen with things like `reset_index` where we can add columns.
             from .dataframe import DataFrame


Test patch: