Repository: modin-project/modin

Issue:
Insert of category column loses category type
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:  Windows 10 1809
- **Modin version** (`modin.__version__`):  0.7.3+41.gd617985
- **Python version**:  Python 3.7.5
- **Code we can use to reproduce**:
```python
if __name__ == "__main__":
    import modin.pandas as pd

    data = {
        "col1": [0, 1, 2, 3],
        "col2": [4, 5, 6, 7],
    }

    df = pd.DataFrame(data).astype({"col1": "category"})

    col1 = df["col1"].copy()
    df = df.drop("col1", axis=1)

    df.insert(0, "col1", col1)
    print(df.dtypes)
```
<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->
The output of the upper code would be:
```
col1    int64
col2    int64
dtype: object
```
When with pandas we get:
```
col1    category
col2       int64
dtype: object
```
So, modin loses category type of column while inserting it.
### Source code / logs
The problem seems to be here: [modin/backends/pandas/query_compiler.py: def insert()](https://github.com/modin-project/modin/blob/master/modin/backends/pandas/query_compiler.py#L980-L986)
Part of function that causes the problem:
 ```python
 if is_list_like(value):
     if isinstance(value, pandas.Series):
         value = value.reindex(self.index)
     value = list(value)
```
`type(col1) == modin.pandas.series.Series` so it is list like object that contains categories, but when we do 
```python
value = list(value)
``` 
it becomes just to a list of ints



Discussion:

Resolving patch:
diff --git a/modin/backends/pandas/query_compiler.py b/modin/backends/pandas/query_compiler.py
--- a/modin/backends/pandas/query_compiler.py
+++ b/modin/backends/pandas/query_compiler.py
@@ -988,7 +988,8 @@ def insert(self, loc, column, value):
             # the data is partitioned identically.
             if isinstance(value, pandas.Series):
                 value = value.reindex(self.index)
-            value = list(value)
+            else:
+                value = list(value)
 
         def insert(df, internal_indices=[]):
             internal_idx = int(internal_indices[0])
diff --git a/modin/engines/base/frame/partition_manager.py b/modin/engines/base/frame/partition_manager.py
--- a/modin/engines/base/frame/partition_manager.py
+++ b/modin/engines/base/frame/partition_manager.py
@@ -16,6 +16,7 @@
 
 from modin.error_message import ErrorMessage
 from modin.data_management.utils import compute_chunksize
+from pandas.api.types import union_categoricals
 
 
 class BaseFrameManager(object):
@@ -223,6 +224,30 @@ def concat(cls, axis, left_parts, right_parts):
         else:
             return np.append(left_parts, right_parts, axis=axis)
 
+    @classmethod
+    def concatenate(cls, dfs):
+        """
+        Concatenate Pandas DataFrames with saving 'category' dtype
+
+        Parameters
+        ----------
+            dfs: list of DataFrames
+
+        Returns
+        -------
+            A Pandas DataFrame
+        """
+        categoricals_columns = set.intersection(
+            *[set(df.select_dtypes("category").columns.tolist()) for df in dfs]
+        )
+
+        for col in categoricals_columns:
+            uc = union_categoricals([df[col] for df in dfs])
+            for df in dfs:
+                df[col] = pandas.Categorical(df[col], categories=uc.categories)
+
+        return pandas.concat(dfs)
+
     @classmethod
     def to_pandas(cls, partitions):
         """Convert this object into a Pandas DataFrame from the partitions.
@@ -251,7 +276,7 @@ def to_pandas(cls, partitions):
         if len(df_rows) == 0:
             return pandas.DataFrame()
         else:
-            return pandas.concat(df_rows)
+            return cls.concatenate(df_rows)
 
     @classmethod
     def to_numpy(cls, partitions):
diff --git a/modin/pandas/dataframe.py b/modin/pandas/dataframe.py
--- a/modin/pandas/dataframe.py
+++ b/modin/pandas/dataframe.py
@@ -903,10 +903,12 @@ def insert(self, loc, column, value, allow_duplicates=False):
             if len(value.columns) != 1:
                 raise ValueError("Wrong number of items passed 2, placement implies 1")
             value = value.iloc[:, 0]
+
+        if isinstance(value, Series):
+            # TODO: Remove broadcast of Series
+            value = value._to_pandas()
+
         if len(self.index) == 0:
-            if isinstance(value, Series):
-                # TODO: Remove broadcast of Series
-                value = value._to_pandas()
             try:
                 value = pandas.Series(value)
             except (TypeError, ValueError, IndexError):


Test patch:
diff --git a/modin/pandas/test/test_dataframe.py b/modin/pandas/test/test_dataframe.py
--- a/modin/pandas/test/test_dataframe.py
+++ b/modin/pandas/test/test_dataframe.py
@@ -64,6 +64,50 @@
 matplotlib.use("Agg")
 
 
+def eval_general(modin_df, pandas_df, operation, comparator=df_equals, **kwargs):
+    md_kwargs, pd_kwargs = {}, {}
+
+    def execute_callable(fn, md_kwargs={}, pd_kwargs={}):
+        try:
+            pd_result = fn(pandas_df, **pd_kwargs)
+        except Exception as e:
+            with pytest.raises(type(e)):
+                fn(modin_df, **md_kwargs)
+        else:
+            md_result = fn(modin_df, **md_kwargs)
+            return md_result, pd_result
+
+    for key, value in kwargs.items():
+        if callable(value):
+            values = execute_callable(value)
+            # that means, that callable raised an exception
+            if values is None:
+                return
+            else:
+                md_value, pd_value = values
+        else:
+            md_value, pd_value = value, value
+
+        md_kwargs[key] = md_value
+        pd_kwargs[key] = pd_value
+
+    values = execute_callable(operation, md_kwargs=md_kwargs, pd_kwargs=pd_kwargs)
+    if values is not None:
+        comparator(*values)
+
+
+def eval_insert(modin_df, pandas_df, **kwargs):
+    _kwargs = {"loc": 0, "col": "New column"}
+    _kwargs.update(kwargs)
+
+    eval_general(
+        modin_df,
+        pandas_df,
+        operation=lambda df, **kwargs: df.insert(**kwargs),
+        **_kwargs,
+    )
+
+
 class TestDataFrameBinary:
     # Test inter df math functions
     def inter_df_math_helper(self, modin_df, pandas_df, op):
@@ -1443,83 +1487,89 @@ def test_dropna_subset_error(self, data):
                 modin_df.dropna(axis=1, subset=[4, 5])
 
     @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
-    @pytest.mark.parametrize("loc", int_arg_values, ids=arg_keys("loc", int_arg_keys))
-    def test_insert(self, data, loc):
-        modin_df = pd.DataFrame(data)
-        pandas_df = pandas.DataFrame(data)
-
-        modin_df = modin_df.copy()
-        pandas_df = pandas_df.copy()
-        column = "New Column"
-        value = modin_df.iloc[:, 0]
-
-        try:
-            pandas_df.insert(loc, column, value)
-        except Exception as e:
-            with pytest.raises(type(e)):
-                modin_df.insert(loc, column, value)
-        else:
-            modin_df.insert(loc, column, value)
-            df_equals(modin_df, pandas_df)
-
-        with pytest.raises(ValueError):
-            modin_df.insert(0, "Bad Column", modin_df)
-
-        modin_df = pd.DataFrame(data)
-        pandas_df = pandas.DataFrame(data)
-
-        modin_df.insert(0, "Duplicate", modin_df[modin_df.columns[0]])
-        pandas_df.insert(0, "Duplicate", pandas_df[pandas_df.columns[0]])
-        df_equals(modin_df, pandas_df)
-
-        modin_df = pd.DataFrame(data)
-        pandas_df = pandas.DataFrame(data)
-
-        modin_df.insert(0, "Scalar", 100)
-        pandas_df.insert(0, "Scalar", 100)
-        df_equals(modin_df, pandas_df)
-
-        with pytest.raises(ValueError):
-            modin_df.insert(0, "Too Short", list(modin_df[modin_df.columns[0]])[:-1])
-
-        with pytest.raises(ValueError):
-            modin_df.insert(0, modin_df.columns[0], modin_df[modin_df.columns[0]])
-
-        with pytest.raises(IndexError):
-            modin_df.insert(len(modin_df.columns) + 100, "Bad Loc", 100)
+    @pytest.mark.parametrize(
+        "astype",
+        [
+            "category",
+            pytest.param(
+                "int32",
+                marks=pytest.mark.xfail(
+                    reason="Modin astype() does not raises ValueError at non-numeric argument when Pandas does."
+                ),
+            ),
+            "float",
+        ],
+    )
+    def test_insert_dtypes(self, data, astype):
+        modin_df, pandas_df = pd.DataFrame(data), pandas.DataFrame(data)
 
-        modin_df = pd.DataFrame(data)
-        pandas_df = pandas.DataFrame(data)
+        # categories with NaN works incorrect for now
+        if astype == "category" and pandas_df.iloc[:, 0].isnull().any():
+            return
 
-        modin_result = pd.DataFrame(columns=list("ab")).insert(
-            0, modin_df.columns[0], modin_df[modin_df.columns[0]]
+        eval_insert(
+            modin_df,
+            pandas_df,
+            col="TypeSaver",
+            value=lambda df: df.iloc[:, 0].astype(astype),
         )
-        pandas_result = pandas.DataFrame(columns=list("ab")).insert(
-            0, pandas_df.columns[0], pandas_df[pandas_df.columns[0]]
-        )
-        df_equals(modin_result, pandas_result)
 
-        modin_df = pd.DataFrame(data)
-        pandas_df = pandas.DataFrame(data)
+    @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
+    @pytest.mark.parametrize("loc", int_arg_values, ids=arg_keys("loc", int_arg_keys))
+    def test_insert_loc(self, data, loc):
+        modin_df, pandas_df = pd.DataFrame(data), pandas.DataFrame(data)
+        value = modin_df.iloc[:, 0]
 
-        modin_result = pd.DataFrame(index=modin_df.index).insert(
-            0, modin_df.columns[0], modin_df[modin_df.columns[0]]
-        )
-        pandas_result = pandas.DataFrame(index=pandas_df.index).insert(
-            0, pandas_df.columns[0], pandas_df[pandas_df.columns[0]]
-        )
-        df_equals(modin_result, pandas_result)
+        eval_insert(modin_df, pandas_df, loc=loc, value=value)
 
-        modin_df = pd.DataFrame(data)
-        pandas_df = pandas.DataFrame(data)
+    @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
+    def test_insert(self, data):
+        modin_df, pandas_df = pd.DataFrame(data), pandas.DataFrame(data)
 
-        modin_result = modin_df.insert(
-            0, "DataFrame insert", modin_df[[modin_df.columns[0]]]
+        eval_insert(
+            modin_df, pandas_df, col="Duplicate", value=lambda df: df[df.columns[0]]
+        )
+        eval_insert(modin_df, pandas_df, col="Scalar", value=100)
+        eval_insert(
+            pd.DataFrame(columns=list("ab")),
+            pandas.DataFrame(columns=list("ab")),
+            col=lambda df: df.columns[0],
+            value=lambda df: df[df.columns[0]],
+        )
+        eval_insert(
+            pd.DataFrame(index=modin_df.index),
+            pandas.DataFrame(index=pandas_df.index),
+            col=lambda df: df.columns[0],
+            value=lambda df: df[df.columns[0]],
+        )
+        eval_insert(
+            modin_df,
+            pandas_df,
+            col="DataFrame insert",
+            value=lambda df: df[[df.columns[0]]],
+        )
+
+        # Bad inserts
+        eval_insert(modin_df, pandas_df, col="Bad Column", value=lambda df: df)
+        eval_insert(
+            modin_df,
+            pandas_df,
+            col="Too Short",
+            value=lambda df: list(df[df.columns[0]])[:-1],
+        )
+        eval_insert(
+            modin_df,
+            pandas_df,
+            col=lambda df: df.columns[0],
+            value=lambda df: df[df.columns[0]],
+        )
+        eval_insert(
+            modin_df,
+            pandas_df,
+            loc=lambda df: len(df.columns) + 100,
+            col="Bad Loc",
+            value=100,
         )
-        pandas_result = pandas_df.insert(
-            0, "DataFrame insert", pandas_df[[pandas_df.columns[0]]]
-        )
-        df_equals(modin_result, pandas_result)
 
     @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
     def test_ndim(self, data):
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -14,7 +14,11 @@
 import copy
 import numpy as np
 import pandas
-from pandas.util.testing import assert_almost_equal, assert_frame_equal
+from pandas.util.testing import (
+    assert_almost_equal,
+    assert_frame_equal,
+    assert_categorical_equal,
+)
 import modin.pandas as pd
 from modin.pandas.utils import to_pandas
 from io import BytesIO
@@ -369,6 +373,19 @@
 ]
 
 
+def df_categories_equals(df1, df2):
+    categories_columns = df1.select_dtypes(include="category").columns
+
+    for column in categories_columns:
+        is_category_ordered = df1[column].dtype.ordered
+        assert_categorical_equal(
+            df1[column].values,
+            df2[column].values,
+            check_dtype=False,
+            check_category_order=is_category_ordered,
+        )
+
+
 def df_equals(df1, df2):
     """Tests if df1 and df2 are equal.
 
@@ -438,6 +455,7 @@ def df_equals(df1, df2):
                 check_datetimelike_compat=True,
                 check_index_type=False,
                 check_column_type=False,
+                check_categorical=False,
             )
         except Exception:
             assert_frame_equal(
@@ -447,7 +465,9 @@ def df_equals(df1, df2):
                 check_datetimelike_compat=True,
                 check_index_type=False,
                 check_column_type=False,
+                check_categorical=False,
             )
+        df_categories_equals(df1, df2)
     elif isinstance(df1, types_for_almost_equals) and isinstance(
         df2, types_for_almost_equals
     ):