Repository: modin-project/modin

Issue:
Expand implementation for DataFrame constructor to make possible construct from dictionaries with Modin entities as values
### Describe the problem
<!-- Describe the problem clearly here. -->

We should expand implementation of DataFrame constructor to make possible to create Modin DataFrame from dictionaries with Modin Series as dict values with fast way. For now we have the follow warning:

```bash
UserWarning: Distributing <class 'dict'> object. This may take some time.
```

```python
from time import time as timer

import numpy as np
# import pandas as pd
import modin.pandas as pd
import ray
ray.init()

nrows = 1000_000_000
df = pd.DataFrame({"a": np.random.rand(nrows), "b": np.random.rand(nrows)})

t = timer()
df2 = pd.DataFrame({"c": df.a})
print(f'df creation time: {timer() - t} s')
```

The result on 112 CPUs, Ray engine:
```bash
df creation time: 3.937314748764038 s   # Pandas is used
df creation time: 24.079696655273438 s # Modin is used
```




Discussion:
What about cpu count? Engine?
> What about cpu count? Engine?

The info related of cpu count was added in the PR description. The execution engine you can see in the reproducer - Ray.
Connected with https://github.com/modin-project/modin/issues/1572
Note that after certain changes where we started to define runtime environment for Ray, we should properly exclude Ray init time from measurement.

The line I personally use to initialize all the workers is this:
```python
pd.DataFrame(range(cfg.CpuCount.get() * cfg.MinPartitionSize().get())).to_numpy() # init the engine and start all the workers
```
The following snippet:
```python
from time import time as timer

import numpy as np
# import pandas as pd
import modin.pandas as pd
import modin.config as cfg

pd.DataFrame(range(cfg.CpuCount.get() * cfg.MinPartitionSize().get())).to_numpy() # init the engine and start all the workers

nrows = 100_000_000
df = pd.DataFrame({"a": np.random.rand(nrows), "b": np.random.rand(nrows)})

t = timer()
df2 = pd.DataFrame({"c": df.a})
print(f'df creation time: {timer() - t} s')
repr(df2)
print(f'df creation + sync time: {timer() - t} s')
```
produces the following timings:
```
df creation time: 1.973231315612793 s
df creation + sync time: 2.2475578784942627 s
```
on 0.15.2 and `MODIN_CPUS=12`

Resolving patch:
diff --git a/modin/pandas/dataframe.py b/modin/pandas/dataframe.py
--- a/modin/pandas/dataframe.py
+++ b/modin/pandas/dataframe.py
@@ -179,6 +179,26 @@ def _init(
             elif is_dict_like(data) and not isinstance(
                 data, (pandas.Series, Series, pandas.DataFrame, DataFrame)
             ):
+                if columns is not None:
+                    data = {key: value for key, value in data.items() if key in columns}
+
+                if len(data) and all(isinstance(v, Series) for v in data.values()):
+                    from .general import concat
+
+                    new_qc = concat(
+                        data.values(), axis=1, keys=data.keys()
+                    )._query_compiler
+
+                    if dtype is not None:
+                        new_qc = new_qc.astype({col: dtype for col in new_qc.columns})
+                    if index is not None:
+                        new_qc = new_qc.reindex(axis=0, labels=index)
+                    if columns is not None:
+                        new_qc = new_qc.reindex(axis=1, labels=columns)
+
+                    self._query_compiler = new_qc
+                    return
+
                 data = {
                     k: v._to_pandas() if isinstance(v, Series) else v
                     for k, v in data.items()


Test patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
@@ -2139,6 +2139,43 @@ def test_shape_hint_detection_from_arrow(self):
         df = pd.utils.from_arrow(at)
         assert df._query_compiler._shape_hint == "column"
 
+    def test_constructor_from_modin_series(self):
+        def construct_has_common_projection(lib, df, **kwargs):
+            return lib.DataFrame({"col1": df.iloc[:, 0], "col2": df.iloc[:, 1]})
+
+        def construct_no_common_projection(lib, df1, df2, **kwargs):
+            return lib.DataFrame(
+                {"col1": df1.iloc[:, 0], "col2": df2.iloc[:, 0], "col3": df1.iloc[:, 1]}
+            )
+
+        def construct_mixed_data(lib, df1, df2, **kwargs):
+            return lib.DataFrame(
+                {
+                    "col1": df1.iloc[:, 0],
+                    "col2": df2.iloc[:, 0],
+                    "col3": df1.iloc[:, 1],
+                    "col4": np.arange(len(df1)),
+                }
+            )
+
+        run_and_compare(
+            construct_has_common_projection, data={"a": [1, 2, 3, 4], "b": [3, 4, 5, 6]}
+        )
+        run_and_compare(
+            construct_no_common_projection,
+            data={"a": [1, 2, 3, 4], "b": [3, 4, 5, 6]},
+            data2={"a": [10, 20, 30, 40]},
+            # HDK doesn't support concatenation of frames that has no common projection
+            force_lazy=False,
+        )
+        run_and_compare(
+            construct_mixed_data,
+            data={"a": [1, 2, 3, 4], "b": [3, 4, 5, 6]},
+            data2={"a": [10, 20, 30, 40]},
+            # HDK doesn't support concatenation of frames that has no common projection
+            force_lazy=False,
+        )
+
 
 class TestArrowExecution:
     data1 = {"a": [1, 2, 3], "b": [3, 4, 5], "c": [6, 7, 8]}
diff --git a/modin/pandas/test/dataframe/test_map_metadata.py b/modin/pandas/test/dataframe/test_map_metadata.py
--- a/modin/pandas/test/dataframe/test_map_metadata.py
+++ b/modin/pandas/test/dataframe/test_map_metadata.py
@@ -1478,3 +1478,40 @@ def test___round__():
     data = test_data_values[0]
     with warns_that_defaulting_to_pandas():
         pd.DataFrame(data).__round__()
+
+
+@pytest.mark.parametrize(
+    "get_index",
+    [
+        pytest.param(lambda idx: None, id="None_idx"),
+        pytest.param(lambda idx: ["a", "b", "c"], id="No_intersection_idx"),
+        pytest.param(lambda idx: idx, id="Equal_idx"),
+        pytest.param(lambda idx: idx[::-1], id="Reversed_idx"),
+    ],
+)
+@pytest.mark.parametrize(
+    "get_columns",
+    [
+        pytest.param(lambda idx: None, id="None_idx"),
+        pytest.param(lambda idx: ["a", "b", "c"], id="No_intersection_idx"),
+        pytest.param(lambda idx: idx, id="Equal_idx"),
+        pytest.param(lambda idx: idx[::-1], id="Reversed_idx"),
+    ],
+)
+@pytest.mark.parametrize("dtype", [None, "str"])
+def test_constructor_from_modin_series(get_index, get_columns, dtype):
+    modin_df, pandas_df = create_test_dfs(test_data_values[0])
+
+    modin_data = {f"new_col{i}": modin_df.iloc[:, i] for i in range(modin_df.shape[1])}
+    pandas_data = {
+        f"new_col{i}": pandas_df.iloc[:, i] for i in range(pandas_df.shape[1])
+    }
+
+    index = get_index(modin_df.index)
+    columns = get_columns(list(modin_data.keys()))
+
+    new_modin = pd.DataFrame(modin_data, index=index, columns=columns, dtype=dtype)
+    new_pandas = pandas.DataFrame(
+        pandas_data, index=index, columns=columns, dtype=dtype
+    )
+    df_equals(new_modin, new_pandas)