Repository: modin-project/modin

Issue:
"TypeError: 'NoneType' object is not iterable" when using pd.concat 
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Ubuntu 18.04.2 LTS
- **Modin installed from (source or binary)**: source
- **Modin version**: 0.4.0
- **Python version**: 3.7
- **Exact command to reproduce**: pd.concat([test_data, row])

### Describe the problem
Using pd.concat to assemble a dataframe from individual rows causes modin to throw the following error. 

### Source code / logs
```
import modin.pandas as pd

test_data = pd.DataFrame()
row = pd.Series({0:"a", 1:"b"})
pd.concat([test_data, row])
```



> ---------------------------------------------------------------------------
> TypeError                                 Traceback (most recent call last)
> <ipython-input-20-1558f0fda4ff> in <module>
>       3 test_data = pd.DataFrame()
>       4 row = pd.Series({0:"a", 1:"b"})
> ----> 5 pd.concat([test_data, row])
> 
> /tmp/res-933/lib/python3.7/site-packages/modin/pandas/concat.py in concat(objs, axis, join, join_axes, ignore_index, keys, levels, names, verify_integrity, sort, copy)
>     116         verify_integrity=False,
>     117         copy=True,
> --> 118         sort=False,
>     119     )
>     120     result_df = DataFrame(query_compiler=new_query_compiler)
> 
> /tmp/res-933/lib/python3.7/site-packages/modin/data_management/query_compiler/pandas_query_compiler.py in concat(self, axis, other, **kwargs)
>     310             Concatenated objects.
>     311         """
> --> 312         return self._append_list_of_managers(other, axis, **kwargs)
>     313 
>     314     def _append_list_of_managers(self, others, axis, **kwargs):
> 
> /tmp/res-933/lib/python3.7/site-packages/modin/data_management/query_compiler/pandas_query_compiler.py in _append_list_of_managers(self, others, axis, **kwargs)
>     323         ignore_index = kwargs.get("ignore_index", False)
>     324         new_self, to_append, joined_axis = self.copartition(
> --> 325             axis ^ 1, others, join, sort, force_repartition=True
>     326         )
>     327         new_data = new_self.concat(axis, to_append)
> 
> /tmp/res-933/lib/python3.7/site-packages/modin/data_management/query_compiler/pandas_query_compiler.py in copartition(self, axis, other, how_to_join, sort, force_repartition)
>     447 
>     448             reindexed_self, reindexed_other = reindexed_self.copartition_datasets(
> --> 449                 axis, other[i].data, reindex_left, reindex_right
>     450             )
>     451             reindexed_other_list.append(reindexed_other)
> 
> /tmp/res-933/lib/python3.7/site-packages/modin/engines/base/block_partitions.py in copartition_datasets(self, axis, other, left_func, right_func)
>     273                 axis,
>     274                 right_func,
> --> 275                 new_self.block_lengths if axis == 0 else new_self.block_widths,
>     276             )
>     277         return new_self, new_other
> 
> /tmp/res-933/lib/python3.7/site-packages/modin/engines/base/block_partitions.py in manual_shuffle(self, axis, shuffle_func, lengths)
>    1035             partitions = self.column_partitions
>    1036         func = self.preprocess_func(shuffle_func)
> -> 1037         result = np.array([part.shuffle(func, lengths) for part in partitions])
>    1038         return self.__constructor__(result) if axis else self.__constructor__(result.T)
>    1039 
> 
> /tmp/res-933/lib/python3.7/site-packages/modin/engines/base/block_partitions.py in <listcomp>(.0)
>    1035             partitions = self.column_partitions
>    1036         func = self.preprocess_func(shuffle_func)
> -> 1037         result = np.array([part.shuffle(func, lengths) for part in partitions])
>    1038         return self.__constructor__(result) if axis else self.__constructor__(result.T)
>    1039 
> 
> /tmp/res-933/lib/python3.7/site-packages/modin/engines/base/axis_partition.py in shuffle(self, func, lengths, **kwargs)
>     159         args = [self.axis, func, num_splits, kwargs, False]
>     160         args.extend(self.list_of_blocks)
> --> 161         return self._wrap_partitions(self.deploy_axis_func(*args))
>     162 
>     163     @classmethod
> 
> /tmp/res-933/lib/python3.7/site-packages/modin/engines/base/axis_partition.py in _wrap_partitions(self, partitions)
>      80             return [self.partition_type(partitions)]
>      81         else:
> ---> 82             return [self.partition_type(obj) for obj in partitions]
>      83 
>      84 
> 
> TypeError: 'NoneType' object is not iterable


Discussion:
Hi @Rockgecko, thanks for posting! This is a bug and should get fixed before the next release.

This bug was introduced in #522 when we added support for `Series`. The `TypeError` given here shouldn't be possible because the `partitions` object in `axis_partition.py` should never be `None`.

Great find, thanks! 

Resolving patch:
diff --git a/modin/pandas/concat.py b/modin/pandas/concat.py
--- a/modin/pandas/concat.py
+++ b/modin/pandas/concat.py
@@ -5,6 +5,7 @@
 import pandas
 from pandas import compat
 from .dataframe import DataFrame
+from .series import Series
 
 
 def concat(
@@ -21,13 +22,14 @@ def concat(
     copy=True,
 ):
     if isinstance(
-        objs, (pandas.Series, DataFrame, compat.string_types, pandas.DataFrame)
+        objs, (pandas.Series, Series, DataFrame, compat.string_types, pandas.DataFrame)
     ):
         raise TypeError(
             "first argument must be an iterable of pandas "
             "objects, you passed an object of type "
             '"{name}"'.format(name=type(objs).__name__)
         )
+    axis = pandas.DataFrame()._get_axis_number(axis)
     objs = list(objs)
     if len(objs) == 0:
         raise ValueError("No objects to concatenate")
@@ -40,7 +42,7 @@ def concat(
         type_check = next(
             obj
             for obj in objs
-            if not isinstance(obj, (pandas.Series, pandas.DataFrame, DataFrame))
+            if not isinstance(obj, (pandas.Series, Series, pandas.DataFrame, DataFrame))
         )
     except StopIteration:
         type_check = None
@@ -52,33 +54,28 @@ def concat(
             "valid",
             type(type_check),
         )
-    all_series = all(isinstance(obj, pandas.Series) for obj in objs)
-    if all_series:
-        return DataFrame(
-            pandas.concat(
-                objs,
+    all_series = all(isinstance(obj, Series) for obj in objs)
+    if all_series and axis == 0:
+        return Series(
+            query_compiler=objs[0]._query_compiler.concat(
                 axis,
-                join,
-                join_axes,
-                ignore_index,
-                keys,
-                levels,
-                names,
-                verify_integrity,
-                sort,
-                copy,
+                [o._query_compiler for o in objs[1:]],
+                join=join,
+                join_axes=None,
+                ignore_index=ignore_index,
+                keys=None,
+                levels=None,
+                names=None,
+                verify_integrity=False,
+                copy=True,
+                sort=sort,
             )
         )
     if isinstance(objs, dict):
-        raise NotImplementedError(
-            "Obj as dicts not implemented. To contribute to "
-            "Modin, please visit github.com/ray-project/ray."
-        )
-    axis = pandas.DataFrame()._get_axis_number(axis)
-
+        raise NotImplementedError("Obj as dicts not implemented.")
     if join not in ["inner", "outer"]:
         raise ValueError(
-            "Only can inner (intersect) or outer (union) join the" " other axis"
+            "Only can inner (intersect) or outer (union) join the other axis"
         )
     # We have the weird Series and axis check because, when concatenating a
     # dataframe to a series on axis=0, pandas ignores the name of the series,
@@ -87,12 +84,11 @@ def concat(
         obj
         if isinstance(obj, DataFrame)
         else DataFrame(obj.rename())
-        if isinstance(obj, pandas.Series) and axis == 0
+        if isinstance(obj, (pandas.Series, Series)) and axis == 0
         else DataFrame(obj)
         for obj in objs
     ]
-    df = objs[0]
-    objs = [obj._query_compiler for obj in objs]
+    objs = [obj._query_compiler for obj in objs if len(obj.index) or len(obj.columns)]
     if keys is not None:
         objs = [objs[i] for i in range(min(len(objs), len(keys)))]
         new_idx_labels = {
@@ -102,7 +98,7 @@ def concat(
         new_idx = pandas.MultiIndex.from_tuples(tuples)
     else:
         new_idx = None
-    new_query_compiler = df._query_compiler.concat(
+    new_query_compiler = objs[0].concat(
         axis,
         objs[1:],
         join=join,
@@ -113,7 +109,7 @@ def concat(
         names=None,
         verify_integrity=False,
         copy=True,
-        sort=False,
+        sort=sort,
     )
     result_df = DataFrame(query_compiler=new_query_compiler)
     if new_idx is not None:


Test patch:
diff --git a/modin/pandas/test/test_concat.py b/modin/pandas/test/test_concat.py
--- a/modin/pandas/test/test_concat.py
+++ b/modin/pandas/test/test_concat.py
@@ -8,6 +8,7 @@
 
 import modin.pandas as pd
 from modin.pandas.utils import from_pandas, to_pandas
+from .utils import df_equals
 
 pd.DEFAULT_NPARTITIONS = 4
 
@@ -186,3 +187,24 @@ def test_concat_non_subscriptable_keys():
     modin_result = pd.concat(modin_dict.values(), keys=modin_dict.keys())
     pandas_result = pandas.concat(pandas_dict.values(), keys=pandas_dict.keys())
     modin_df_equals_pandas(modin_result, pandas_result)
+
+
+def test_concat_series_only():
+    modin_series = pd.Series(list(range(1000)))
+    pandas_series = pandas.Series(list(range(1000)))
+
+    df_equals(
+        pd.concat([modin_series, modin_series]),
+        pandas.concat([pandas_series, pandas_series]),
+    )
+
+
+def test_concat_with_empty_frame():
+    modin_empty_df = pd.DataFrame()
+    pandas_empty_df = pandas.DataFrame()
+    modin_row = pd.Series({0: "a", 1: "b"})
+    pandas_row = pandas.Series({0: "a", 1: "b"})
+    df_equals(
+        pd.concat([modin_empty_df, modin_row]),
+        pandas.concat([pandas_empty_df, pandas_row]),
+    )