Repository: modin-project/modin

Issue:
Unable to perform head, tail,sample on dataframe
This is what I did to check out modin as a pandas replacement 

import modin.pandas as pd
d = pd.read_csv('boston_housing.csv')
d.head()

and I got the following error:
Traceback (most recent call last):
  File "/root/anaconda3/lib/python3.6/site-packages/modin/pandas/utils.py", line 380, in create_blocks
    return _create_blocks_helper(df, npartitions, axis)
NameError: name '_create_blocks_helper' is not defined

any clue as to why this might have come


Discussion:
Hi @alokgogate, thanks for reporting this issue! 

- How did you install modin? 
- What operating system are you running on? 
- What version of python are you running?

This is an [internal function](https://github.com/modin-project/modin/blob/398a4a1da5c97ce01f7d88f82dc4e18b2ebf68bf/modin/pandas/utils.py#L383), so it shouldn't happen. Did you edit any of the source files?

Resolving patch:
diff --git a/modin/pandas/io.py b/modin/pandas/io.py
--- a/modin/pandas/io.py
+++ b/modin/pandas/io.py
@@ -56,13 +56,6 @@ def read_parquet(path, engine='auto', columns=None, **kwargs):
     return DataFrame(block_partitions=blk_partitions, columns=columns)
 
 
-@ray.remote
-def _read_parquet_column(path, column, kwargs={}):
-    df = pq.read_pandas(path, columns=[column], **kwargs).to_pandas()
-    oids = _partition_pandas_dataframe(df, num_partitions=get_npartitions())
-    return oids
-
-
 # CSV
 def _skip_header(f, kwargs={}):
     lines_read = 0
@@ -183,19 +176,6 @@ def _read_csv_from_file(filepath, npartitions, kwargs={}):
     return df
 
 
-@ray.remote
-def _read_csv_with_offset(fn, start, end, kwargs={}, header=b''):
-    bio = open(fn, 'rb')
-    bio.seek(start)
-    to_read = header + bio.read(end - start)
-    bio.close()
-    pandas_df = pandas.read_csv(BytesIO(to_read), **kwargs)
-    index = pandas_df.index
-    # Partitions must have RangeIndex
-    pandas_df.index = pandas.RangeIndex(0, len(pandas_df))
-    return pandas_df, index
-
-
 def _read_csv_from_pandas(filepath_or_buffer, kwargs):
     pd_obj = pandas.read_csv(filepath_or_buffer, **kwargs)
 
@@ -211,13 +191,6 @@ def _read_csv_from_pandas(filepath_or_buffer, kwargs):
     return pd_obj
 
 
-@ray.remote
-def get_index(index_name, *partition_indices):
-    index = partition_indices[0].append(partition_indices[1:])
-    index.names = index_name
-    return index
-
-
 def read_csv(filepath_or_buffer,
              sep=',',
              delimiter=None,
@@ -598,3 +571,30 @@ def read_sql(sql,
     ray_frame = from_pandas(port_frame, get_npartitions())
 
     return ray_frame
+
+
+@ray.remote
+def get_index(index_name, *partition_indices):
+    index = partition_indices[0].append(partition_indices[1:])
+    index.names = index_name
+    return index
+
+
+@ray.remote
+def _read_csv_with_offset(fn, start, end, kwargs={}, header=b''):
+    bio = open(fn, 'rb')
+    bio.seek(start)
+    to_read = header + bio.read(end - start)
+    bio.close()
+    pandas_df = pandas.read_csv(BytesIO(to_read), **kwargs)
+    index = pandas_df.index
+    # Partitions must have RangeIndex
+    pandas_df.index = pandas.RangeIndex(0, len(pandas_df))
+    return pandas_df, index
+
+
+@ray.remote
+def _read_parquet_column(path, column, kwargs={}):
+    df = pq.read_pandas(path, columns=[column], **kwargs).to_pandas()
+    oids = _partition_pandas_dataframe(df, num_partitions=get_npartitions())
+    return oids
diff --git a/modin/pandas/utils.py b/modin/pandas/utils.py
--- a/modin/pandas/utils.py
+++ b/modin/pandas/utils.py
@@ -263,30 +263,6 @@ def to_pandas(df):
     return pandas_df
 
 
-@ray.remote
-def extractor(df_chunk, row_loc, col_loc):
-    """Retrieve an item from remote block
-    """
-    # We currently have to do the writable flag trick because a pandas bug
-    # https://github.com/pandas-dev/pandas/issues/17192
-    try:
-        row_loc.flags.writeable = True
-        col_loc.flags.writeable = True
-    except AttributeError:
-        # Locators might be scaler or python list
-        pass
-    return df_chunk.iloc[row_loc, col_loc]
-
-
-@ray.remote
-def writer(df_chunk, row_loc, col_loc, item):
-    """Make a copy of the block and write new item to it
-    """
-    df_chunk = df_chunk.copy()
-    df_chunk.iloc[row_loc, col_loc] = item
-    return df_chunk
-
-
 def _mask_block_partitions(blk_partitions, row_metadata, col_metadata):
     """Return the squeezed/expanded block partitions as defined by
     row_metadata and col_metadata.
@@ -315,21 +291,6 @@ def _mask_block_partitions(blk_partitions, row_metadata, col_metadata):
     return np.array(result_oids).reshape(shape)
 
 
-@ray.remote
-def _deploy_func(func, dataframe, *args):
-    """Deploys a function for the _map_partitions call.
-    Args:
-        dataframe (pandas.DataFrame): The pandas DataFrame for this partition.
-    Returns:
-        A futures object representing the return value of the function
-        provided.
-    """
-    if len(args) == 0:
-        return func(dataframe)
-    else:
-        return func(dataframe, *args)
-
-
 def _map_partitions(func, partitions, *argslists):
     """Apply a function across the specified axis
 
@@ -355,36 +316,6 @@ def _map_partitions(func, partitions, *argslists):
                 for args in zip(partitions, *argslists)]
 
 
-@ray.remote
-def _build_col_widths(df_col):
-    """Compute widths (# of columns) for each partition."""
-    widths = np.array(ray.get([_deploy_func.remote(_get_widths, d)
-                      for d in df_col]))
-
-    return widths
-
-
-@ray.remote
-def _build_row_lengths(df_row):
-    """Compute lengths (# of rows) for each partition."""
-    lengths = np.array(ray.get([_deploy_func.remote(_get_lengths, d)
-                       for d in df_row]))
-
-    return lengths
-
-
-@ray.remote
-def _build_coord_df(lengths, index):
-    """Build the coordinate DataFrame over all partitions."""
-    filtered_lengths = [x for x in lengths if x > 0]
-    coords = None
-    if len(filtered_lengths) > 0:
-        coords = np.vstack([np.column_stack((np.full(l, i), np.arange(l)))
-                            for i, l in enumerate(filtered_lengths)])
-    col_names = ("partition", "index_within_partition")
-    return pandas.DataFrame(coords, index=index, columns=col_names)
-
-
 def _create_block_partitions(partitions, axis=0, length=None):
 
     if length is not None and length != 0 and get_npartitions() > length:
@@ -408,11 +339,6 @@ def _create_block_partitions(partitions, axis=0, length=None):
     return _fix_blocks_dimensions(blocks, axis)
 
 
-@ray.remote
-def create_blocks(df, npartitions, axis):
-    return _create_blocks_helper(df, npartitions, axis)
-
-
 def _create_blocks_helper(df, npartitions, axis):
     # Single partition dataframes don't need to be repartitioned
     if npartitions == 1:
@@ -437,6 +363,123 @@ def _create_blocks_helper(df, npartitions, axis):
     return blocks
 
 
+def _inherit_docstrings(parent, excluded=[]):
+    """Creates a decorator which overwrites a decorated class' __doc__
+    attribute with parent's __doc__ attribute. Also overwrites __doc__ of
+    methods and properties defined in the class with the __doc__ of matching
+    methods and properties in parent.
+
+    Args:
+        parent (object): Class from which the decorated class inherits __doc__.
+        excluded (list): List of parent objects from which the class does not
+            inherit docstrings.
+
+    Returns:
+        function: decorator which replaces the decorated class' documentation
+            parent's documentation.
+    """
+    def decorator(cls):
+        if parent not in excluded:
+            cls.__doc__ = parent.__doc__
+        for attr, obj in cls.__dict__.items():
+            parent_obj = getattr(parent, attr, None)
+            if parent_obj in excluded or \
+                    (not callable(parent_obj) and
+                     not isinstance(parent_obj, property)):
+                continue
+            if callable(obj):
+                obj.__doc__ = parent_obj.__doc__
+            elif isinstance(obj, property) and obj.fget is not None:
+                p = property(obj.fget, obj.fset, obj.fdel, parent_obj.__doc__)
+                setattr(cls, attr, p)
+
+        return cls
+
+    return decorator
+
+
+def _fix_blocks_dimensions(blocks, axis):
+    """Checks that blocks is 2D, and adds a dimension if not.
+    """
+    if blocks.ndim < 2:
+        return np.expand_dims(blocks, axis=axis ^ 1)
+    return blocks
+
+
+@ray.remote
+def _deploy_func(func, dataframe, *args):
+    """Deploys a function for the _map_partitions call.
+    Args:
+        dataframe (pandas.DataFrame): The pandas DataFrame for this partition.
+    Returns:
+        A futures object representing the return value of the function
+        provided.
+    """
+    if len(args) == 0:
+        return func(dataframe)
+    else:
+        return func(dataframe, *args)
+
+
+@ray.remote
+def extractor(df_chunk, row_loc, col_loc):
+    """Retrieve an item from remote block
+    """
+    # We currently have to do the writable flag trick because a pandas bug
+    # https://github.com/pandas-dev/pandas/issues/17192
+    try:
+        row_loc.flags.writeable = True
+        col_loc.flags.writeable = True
+    except AttributeError:
+        # Locators might be scaler or python list
+        pass
+    return df_chunk.iloc[row_loc, col_loc]
+
+
+@ray.remote
+def writer(df_chunk, row_loc, col_loc, item):
+    """Make a copy of the block and write new item to it
+    """
+    df_chunk = df_chunk.copy()
+    df_chunk.iloc[row_loc, col_loc] = item
+    return df_chunk
+
+
+@ray.remote
+def _build_col_widths(df_col):
+    """Compute widths (# of columns) for each partition."""
+    widths = np.array(ray.get([_deploy_func.remote(_get_widths, d)
+                      for d in df_col]))
+
+    return widths
+
+
+@ray.remote
+def _build_row_lengths(df_row):
+    """Compute lengths (# of rows) for each partition."""
+    lengths = np.array(ray.get([_deploy_func.remote(_get_lengths, d)
+                       for d in df_row]))
+
+    return lengths
+
+
+@ray.remote
+def _build_coord_df(lengths, index):
+    """Build the coordinate DataFrame over all partitions."""
+    filtered_lengths = [x for x in lengths if x > 0]
+    coords = None
+    if len(filtered_lengths) > 0:
+        coords = np.vstack([np.column_stack((np.full(l, i), np.arange(l)))
+                            for i, l in enumerate(filtered_lengths)])
+    col_names = ("partition", "index_within_partition")
+    return pandas.DataFrame(coords, index=index, columns=col_names)
+
+
+@ray.remote
+def create_blocks(df, npartitions, axis):
+    return _create_blocks_helper(df, npartitions, axis)
+
+
 @memoize
 @ray.remote
 def _blocks_to_series(*partition):
@@ -474,41 +517,6 @@ def _blocks_to_row(*partition):
         return pandas.DataFrame()
 
 
-def _inherit_docstrings(parent, excluded=[]):
-    """Creates a decorator which overwrites a decorated class' __doc__
-    attribute with parent's __doc__ attribute. Also overwrites __doc__ of
-    methods and properties defined in the class with the __doc__ of matching
-    methods and properties in parent.
-
-    Args:
-        parent (object): Class from which the decorated class inherits __doc__.
-        excluded (list): List of parent objects from which the class does not
-            inherit docstrings.
-
-    Returns:
-        function: decorator which replaces the decorated class' documentation
-            parent's documentation.
-    """
-    def decorator(cls):
-        if parent not in excluded:
-            cls.__doc__ = parent.__doc__
-        for attr, obj in cls.__dict__.items():
-            parent_obj = getattr(parent, attr, None)
-            if parent_obj in excluded or \
-                    (not callable(parent_obj) and
-                     not isinstance(parent_obj, property)):
-                continue
-            if callable(obj):
-                obj.__doc__ = parent_obj.__doc__
-            elif isinstance(obj, property) and obj.fget is not None:
-                p = property(obj.fget, obj.fset, obj.fdel, parent_obj.__doc__)
-                setattr(cls, attr, p)
-
-        return cls
-
-    return decorator
-
-
 @ray.remote
 def _reindex_helper(old_index, new_index, axis, npartitions, *df):
     """Reindexes a DataFrame to prepare for join/concat.
@@ -613,14 +621,6 @@ def _concat_index(*index_parts):
     return index_parts[0].append(index_parts[1:])
 
 
-def _fix_blocks_dimensions(blocks, axis):
-    """Checks that blocks is 2D, and adds a dimension if not.
-    """
-    if blocks.ndim < 2:
-        return np.expand_dims(blocks, axis=axis ^ 1)
-    return blocks
-
-
 @ray.remote
 def _compile_remote_dtypes(*column_of_blocks):
     small_dfs = [df.loc[0:0] for df in column_of_blocks]


Test patch: