Repository: modin-project/modin

Issue:
TypeError: '>' not supported between instances of 'list' and 'int' in iloc
### System information
- **OS Platform and Distribution**: "Ubuntu 18.04 LTS"
- **Modin installed from (source or binary)**: binary
- **Modin version**: 0.1.0
- **Python version**: 3.6.5
- **Exact command to reproduce**: df.iloc[0:5,:]

Calling: **df.iloc[0:5,:]** results in the following error:

Traceback (most recent call last):
  File "bakeoffModin.py", line 15, in <module>
    print( f'Sample:\n {df.iloc[0:5,:]} ')
  File "/usr/local/lib/python3.6/dist-packages/modin/pandas/dataframe.py", line 230, in __str__
    return repr(self)
  File "/usr/local/lib/python3.6/dist-packages/modin/pandas/dataframe.py", line 325, in __repr__
    return repr(self._repr_helper_())
  File "/usr/local/lib/python3.6/dist-packages/modin/pandas/dataframe.py", line 235, in _repr_helper_
    return to_pandas(self)
  File "/usr/local/lib/python3.6/dist-packages/modin/pandas/utils.py", line 227, in to_pandas
    pandas_df = pandas.concat(ray.get(df._row_partitions), copy=False)
  File "/usr/local/lib/python3.6/dist-packages/modin/pandas/dataframe.py", line 198, in _get_row_partitions
    empty_rows_mask = self._row_metadata._lengths > 0
TypeError: '>' not supported between instances of 'list' and 'int'


Discussion:
Hi @Bidek56! Thanks for catching this bug! We will try to get this fixed by the end of the week.

In the meantime, we should still support `df.head(n)`

Resolving patch:
diff --git a/modin/pandas/indexing.py b/modin/pandas/indexing.py
--- a/modin/pandas/indexing.py
+++ b/modin/pandas/indexing.py
@@ -11,10 +11,12 @@
 import ray
 from warnings import warn
 
-from .utils import (_get_nan_block_id, extractor, _mask_block_partitions,
-                    writer, _blocks_to_series)
+from .utils import (_get_nan_block_id, extractor, _repartition_coord_df,
+                    _generate_blocks, _mask_block_partitions, writer,
+                    _blocks_to_series)
 from .index_metadata import _IndexMetadata
 from .dataframe import DataFrame
+from . import get_npartitions
 """Indexing Helper Class works as follows:
 
 _Location_Indexer_Base provide methods framework for __getitem__
@@ -70,6 +72,11 @@ def is_integer_slice(x):
 INCLUDED, END point is EXCLUDED), listlike of integers, boolean array] types.
 """
 
+_VIEW_IS_COPY_WARNING = """
+Modin is making a copy of of the DataFrame. This behavior diverges from Pandas.
+This will be fixed in future releases.
+"""
+
 
 def _parse_tuple(tup):
     """Unpack the user input for getitem and setitem and compute ndim
@@ -153,7 +160,7 @@ def __getitem__(self, row_lookup, col_lookup, ndim):
             ndim: the dimension of returned data
         """
         if ndim == 2:
-            return self._generate_view(row_lookup, col_lookup)
+            return self._generate_view_copy(row_lookup, col_lookup)
 
         extracted = self._retrive_items(row_lookup, col_lookup)
         if ndim == 1:
@@ -193,8 +200,44 @@ def _retrive_items(self, row_lookup, col_lookup):
                 result_oids.append(result_oid)
         return result_oids
 
+    def _generate_view_copy(self, row_lookup, col_lookup):
+        """Generate a new DataFrame by making copies.
+
+        Note (simon):
+            - This is a temporary replacement for _generate_view
+              function below.
+        """
+        warn(_VIEW_IS_COPY_WARNING)
+
+        row_lookup_new = _repartition_coord_df(row_lookup, get_npartitions())
+        col_lookup_new = _repartition_coord_df(col_lookup, get_npartitions())
+
+        new_blocks = _generate_blocks(row_lookup, row_lookup_new, col_lookup,
+                                      col_lookup_new, self.block_oids)
+
+        row_lengths_oid = ray.put(np.bincount(row_lookup_new['partition']))
+        col_lengths_oid = ray.put(np.bincount(col_lookup_new['partition']))
+
+        new_row_metadata = _IndexMetadata(
+            coord_df_oid=row_lookup_new, lengths_oid=row_lengths_oid)
+
+        new_col_metadata = _IndexMetadata(
+            coord_df_oid=col_lookup_new, lengths_oid=col_lengths_oid)
+
+        df_view = DataFrame(
+            block_partitions=new_blocks,
+            row_metadata=new_row_metadata,
+            col_metadata=new_col_metadata,
+            index=row_lookup.index,
+            columns=col_lookup.index)
+
+        return df_view
+
     def _generate_view(self, row_lookup, col_lookup):
         """Generate a DataFrameView from lookup
+
+        Note (simon):
+            - This is not used because of index metadata was broken
         """
         row_lengths = [0] * len(self.df._row_metadata._lengths)
         for i in row_lookup["partition"]:
@@ -203,11 +246,14 @@ def _generate_view(self, row_lookup, col_lookup):
         for i in col_lookup["partition"]:
             col_lengths[i] += 1
 
+        row_lengths_oid = ray.put(np.array(row_lengths))
+        col_lengths_oid = ray.put(np.array(col_lengths))
+
         row_metadata_view = _IndexMetadata(
-            coord_df_oid=row_lookup, lengths_oid=row_lengths)
+            coord_df_oid=row_lookup, lengths_oid=row_lengths_oid)
 
         col_metadata_view = _IndexMetadata(
-            coord_df_oid=col_lookup, lengths_oid=col_lengths)
+            coord_df_oid=col_lookup, lengths_oid=col_lengths_oid)
 
         df_view = DataFrameView(
             block_partitions=self.block_oids,
@@ -368,8 +414,10 @@ def _enlarge_axis(self, locator, axis):
 
         lens = major_meta._lengths
         lens = np.concatenate([lens, np.array([num_nan_labels])])
+        lens_oid = ray.put(np.array(lens))
 
-        metadata_view = _IndexMetadata(coord_df_oid=coord_df, lengths_oid=lens)
+        metadata_view = _IndexMetadata(
+            coord_df_oid=coord_df, lengths_oid=lens_oid)
         return metadata_view
 
     def _compute_enlarge_labels(self, locator, base_index):
@@ -454,6 +502,8 @@ def _check_dtypes(self, locator):
 
 class DataFrameView(DataFrame):
     """A subclass of DataFrame where the index can be smaller than blocks.
+
+    Deprecated because _generate_view_copy is used instead of _generate_view
     """
 
     def __init__(self, block_partitions, row_metadata, col_metadata, index,
diff --git a/modin/pandas/utils.py b/modin/pandas/utils.py
--- a/modin/pandas/utils.py
+++ b/modin/pandas/utils.py
@@ -263,6 +263,107 @@ def to_pandas(df):
     return pandas_df
 
 
+"""
+Indexing Section
+    Generate View Copy Helpers
+    Function list:
+        - `extract_block` (ray.remote function, move to EOF)
+        - `_generate_block`
+        - `_repartition_coord_df`
+    Call Dependency:
+        - _generate_block calls extract_block remote
+    Pipeline:
+        - Repartition the dataframe by npartition
+        - Use case:
+              The dataframe is a DataFrameView, the two coord_dfs only
+              describe the subset of the block partition data. We want
+              to create a new copy of this subset and re-partition
+              the new dataframe.
+"""
+
+
+def _repartition_coord_df(old_coord_df, npartition):
+    """Repartition the (view of) coord_df by npartition
+
+    This function is best used when old_coord_df is not contigous.
+    For example, it turns:
+
+        partition index_within_partition
+    i0  0         0
+    i6  3         2
+
+    into
+
+        partition index_within_partition
+    i0  0         0
+    i6  0         1
+
+    Note(simon):
+        The resulting npartition will be <= npartition
+        passed in.
+    """
+    length = len(old_coord_df)
+    chunksize = (len(old_coord_df) // npartition
+                 if len(old_coord_df) % npartition == 0 else
+                 len(old_coord_df) // npartition + 1)
+
+    # genereate array([0, 0, 0, 1, 1, 1, 2])
+    partitions = np.repeat(np.arange(npartition), chunksize)[:length]
+
+    # generate array([0, 1, 2, 0, 1, 2, 0])
+    final_n_partition = np.max(partitions)
+    idx_in_part = np.tile(np.arange(chunksize), final_n_partition + 1)[:length]
+
+    final_df = pandas.DataFrame(
+        {
+            'partition': partitions,
+            'index_within_partition': idx_in_part
+        },
+        index=old_coord_df.index)
+
+    return final_df
+
+
+def _generate_blocks(old_row, new_row, old_col, new_col,
+                     block_partition_2d_oid_arr):
+    """
+    Given the four coord_dfs:
+        - Old Row Coord df
+        - New Row Coord df
+        - Old Col Coord df
+        - New Col Coord df
+    and the block partition array, this function will generate the new
+    block partition array.
+    """
+
+    # We join the old and new coord_df to find out which chunk in the old
+    # partition belongs to the chunk in the new partition. The new coord df
+    # should have the same index as the old coord df in order to align the
+    # row/column. This is guaranteed by _repartition_coord_df.
+    def join(old, new):
+        return new.merge(
+            old, left_index=True, right_index=True, suffixes=('_new', '_old'))
+
+    row_grouped = join(old_row, new_row).groupby('partition_new')
+    col_grouped = join(old_col, new_col).groupby('partition_new')
+
+    oid_lst = []
+    for row_idx, row_lookup in row_grouped:
+        for col_idx, col_lookup in col_grouped:
+            oid = extract_block.remote(
+                block_partition_2d_oid_arr,
+                row_lookup,
+                col_lookup,
+                col_name_suffix='_old')
+            oid_lst.append(oid)
+    return np.array(oid_lst).reshape(len(row_grouped), len(col_grouped))
+
+
+# Indexing
+#  Generate View Copy Helpers
+# END
+
+
 def _mask_block_partitions(blk_partitions, row_metadata, col_metadata):
     """Return the squeezed/expanded block partitions as defined by
     row_metadata and col_metadata.
@@ -640,3 +741,38 @@ def _concat_index(*index_parts):
 def _compile_remote_dtypes(*column_of_blocks):
     small_dfs = [df.loc[0:0] for df in column_of_blocks]
     return pandas.concat(small_dfs, copy=False).dtypes
+
+
+@ray.remote
+def extract_block(blk_partitions, row_lookup, col_lookup, col_name_suffix):
+    """
+    This function extracts a single block from blk_partitions using
+    the row_lookup and col_lookup.
+
+    Pass in col_name_suffix='_old' when operate on a joined df.
+    """
+
+    def apply_suffix(s):
+        return s + col_name_suffix
+
+    # Address Arrow Error:
+    #   Buffer source array is read-only
+    row_lookup = row_lookup.copy()
+    col_lookup = col_lookup.copy()
+
+    df_columns = []
+    for row_idx, row_df in row_lookup.groupby(apply_suffix('partition')):
+        this_column = []
+        for col_idx, col_df in col_lookup.groupby(apply_suffix('partition')):
+            block_df_oid = blk_partitions[row_idx, col_idx]
+            block_df = ray.get(block_df_oid)
+            chunk = block_df.iloc[row_df[apply_suffix(
+                'index_within_partition')], col_df[apply_suffix(
+                    'index_within_partition')]]
+            this_column.append(chunk)
+        df_columns.append(pandas.concat(this_column, axis=1))
+    final_df = pandas.concat(df_columns)
+    final_df.index = pandas.RangeIndex(0, final_df.shape[0])
+    final_df.columns = pandas.RangeIndex(0, final_df.shape[1])
+
+    return final_df


Test patch:
diff --git a/modin/pandas/test/test_dataframe.py b/modin/pandas/test/test_dataframe.py
--- a/modin/pandas/test/test_dataframe.py
+++ b/modin/pandas/test/test_dataframe.py
@@ -143,7 +143,7 @@ def test_int_dataframe():
     test_cumsum(ray_df, pandas_df)
     test_pipe(ray_df, pandas_df)
 
-    # test_loc(ray_df, pandas_df)
+    test_loc(ray_df, pandas_df)
     test_iloc(ray_df, pandas_df)
 
     labels = ['a', 'b', 'c', 'd']
@@ -313,7 +313,7 @@ def test_float_dataframe():
     test_iteritems(ray_df, pandas_df)
     test_itertuples(ray_df, pandas_df)
 
-    # test_loc(ray_df, pandas_df)
+    test_loc(ray_df, pandas_df)
     test_iloc(ray_df, pandas_df)
 
     labels = ['a', 'b', 'c', 'd']
@@ -495,7 +495,7 @@ def test_mixed_dtype_dataframe():
     test_iteritems(ray_df, pandas_df)
     test_itertuples(ray_df, pandas_df)
 
-    # test_loc(ray_df, pandas_df)
+    test_loc(ray_df, pandas_df)
     test_iloc(ray_df, pandas_df)
 
     labels = ['a', 'b', 'c', 'd']
@@ -646,7 +646,7 @@ def test_nan_dataframe():
     test_iteritems(ray_df, pandas_df)
     test_itertuples(ray_df, pandas_df)
 
-    # test_loc(ray_df, pandas_df)
+    test_loc(ray_df, pandas_df)
     test_iloc(ray_df, pandas_df)
 
     labels = ['a', 'b', 'c', 'd']
@@ -3445,18 +3445,20 @@ def test___repr__():
 
 @pytest.fixture
 def test_loc(ray_df, pd_df):
-    # Singleton
-    assert ray_df.loc[0].equals(pd_df.loc[0])
+    # Scaler
     assert ray_df.loc[0, 'col1'] == pd_df.loc[0, 'col1']
 
-    # List
-    assert ray_df.loc[[1, 2]].equals(pd_df.loc[[1, 2]])
-    assert ray_df.loc[[1, 2], ['col1']].equals(pd_df.loc[[1, 2], ['col1']])
-
-    # Slice
+    # Series
+    assert ray_df.loc[0].equals(pd_df.loc[0])
     assert ray_df.loc[1:, 'col1'].equals(pd_df.loc[1:, 'col1'])
     assert ray_df.loc[1:2, 'col1'].equals(pd_df.loc[1:2, 'col1'])
-    assert ray_df.loc[1:2, 'col1':'col2'].equals(pd_df.loc[1:2, 'col1':'col2'])
+
+    # DataFrame
+    assert ray_df_equals_pandas(ray_df.loc[[1, 2]], pd_df.loc[[1, 2]])
+    assert ray_df_equals_pandas(ray_df.loc[[1, 2], ['col1']],
+                                pd_df.loc[[1, 2], ['col1']])
+    assert ray_df_equals_pandas(ray_df.loc[1:2, 'col1':'col2'],
+                                pd_df.loc[1:2, 'col1':'col2'])
 
 
 def test_is_copy():
@@ -3486,18 +3488,22 @@ def test_ix():
 
 @pytest.fixture
 def test_iloc(ray_df, pd_df):
-    # Singleton
-    assert ray_df.iloc[0].equals(pd_df.iloc[0])
+    # Scaler
     assert ray_df.iloc[0, 1] == pd_df.iloc[0, 1]
 
-    # List
-    assert ray_df.iloc[[1, 2]].equals(pd_df.iloc[[1, 2]])
-    assert ray_df.iloc[[1, 2], [1, 0]].equals(pd_df.iloc[[1, 2], [1, 0]])
-
-    # Slice
+    # Series
+    assert ray_df.iloc[0].equals(pd_df.iloc[0])
     assert ray_df.iloc[1:, 0].equals(pd_df.iloc[1:, 0])
     assert ray_df.iloc[1:2, 0].equals(pd_df.iloc[1:2, 0])
-    assert ray_df.iloc[1:2, 0:2].equals(pd_df.iloc[1:2, 0:2])
+
+    # DataFrame
+    assert ray_df_equals_pandas(ray_df.iloc[[1, 2]], pd_df.iloc[[1, 2]])
+    assert ray_df_equals_pandas(ray_df.iloc[[1, 2], [1, 0]],
+                                pd_df.iloc[[1, 2], [1, 0]])
+    assert ray_df_equals_pandas(ray_df.iloc[1:2, 0:2], pd_df.iloc[1:2, 0:2])
+
+    # Issue #43
+    ray_df.iloc[0:3, :]
 
 
 def test__doc__():