Repository: mars-project/mars

Issue:
[BUG] Fail to read csv when pyarrow not installed
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

Fail to read csv when pyarrow not installed.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

```
File "<ipython-input-34-f537410e66a8>", line 1, in <module>
runfile('C:/Users/tony/CrownVirusAny.py', wdir='C:/Users/tony')

File "C:\Users\tony\Anadata\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 786, in runfile
execfile(filename, namespace)

File "C:\Users\tony\Anadata\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 110, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)

File "C:/Users/tony/CrownVirusAny.py", line 13, in <module>
cf=md.read_csv('countries-aggregated.csv')

File "C:\Users\tony\Anadata\lib\site-packages\mars\dataframe\datasource\read_csv.py", line 552, in read_csv
with open_file(file_path, compression=compression, storage_options=storage_options) as f:

File "C:\Users\tony\Anadata\lib\site-packages\mars\filesystem.py", line 120, in open_file
f = fs.open(path, mode=mode)

AttributeError: 'LocalFileSystem' object has no attribute 'open'
```


Discussion:

Resolving patch:
diff --git a/mars/dataframe/datasource/read_csv.py b/mars/dataframe/datasource/read_csv.py
--- a/mars/dataframe/datasource/read_csv.py
+++ b/mars/dataframe/datasource/read_csv.py
@@ -64,7 +64,7 @@ def _find_hdfs_start_end(f, offset, size):
 
 
 def _find_chunk_start_end(f, offset, size):
-    if isinstance(f, HdfsFile):
+    if HdfsFile is not None and isinstance(f, HdfsFile):
         return _find_hdfs_start_end(f, offset, size)
     f.seek(offset)
     if f.tell() == 0:
diff --git a/mars/filesystem.py b/mars/filesystem.py
--- a/mars/filesystem.py
+++ b/mars/filesystem.py
@@ -14,19 +14,273 @@
 
 import os
 import shutil
-from collections.abc import MutableMapping
+import pathlib
 import glob as local_glob
+from os.path import join as pjoin
+from collections.abc import MutableMapping
 from gzip import GzipFile
 from urllib.parse import urlparse
 
+
 try:
-    from pyarrow import FileSystem
-    from pyarrow import LocalFileSystem as ArrowLocalFileSystem
-    from pyarrow import HadoopFileSystem
+    from pyarrow import FileSystem, LocalFileSystem
     from pyarrow.util import implements
 except ImportError:  # pragma: no cover
-    FileSystem, ArrowLocalFileSystem, HadoopFileSystem = object, object, object
-    implements = None
+    def implements(f):
+        def decorator(g):
+            g.__doc__ = f.__doc__
+            return g
+
+        return decorator
+
+    def _stringify_path(path):
+        """
+        Convert *path* to a string or unicode path if possible.
+        """
+        if isinstance(path, str):
+            return path
+
+        # checking whether path implements the filesystem protocol
+        try:
+            return path.__fspath__()  # new in python 3.6
+        except AttributeError:
+            # fallback pathlib ckeck for earlier python versions than 3.6
+            if isinstance(path, pathlib.Path):
+                return str(path)
+
+        raise TypeError("not a path-like object")
+
+    class FileSystem(object):
+        """
+        Abstract filesystem interface
+        """
+
+        def cat(self, path):
+            """
+            Return contents of file as a bytes object
+
+            Returns
+            -------
+            contents : bytes
+            """
+            with self.open(path, 'rb') as f:
+                return f.read()
+
+        def ls(self, path):
+            """
+            Return list of file paths
+            """
+            raise NotImplementedError
+
+        def delete(self, path, recursive=False):
+            """
+            Delete the indicated file or directory
+
+            Parameters
+            ----------
+            path : string
+            recursive : boolean, default False
+                If True, also delete child paths for directories
+            """
+            raise NotImplementedError
+
+        def disk_usage(self, path):
+            """
+            Compute bytes used by all contents under indicated path in file tree
+
+            Parameters
+            ----------
+            path : string
+                Can be a file path or directory
+
+            Returns
+            -------
+            usage : int
+            """
+            path = _stringify_path(path)
+            path_info = self.stat(path)
+            if path_info['kind'] == 'file':
+                return path_info['size']
+
+            total = 0
+            for root, directories, files in self.walk(path):
+                for child_path in files:
+                    abspath = self._path_join(root, child_path)
+                    total += self.stat(abspath)['size']
+
+            return total
+
+        def _path_join(self, *args):
+            return self.pathsep.join(args)
+
+        def stat(self, path):
+            """
+
+            Returns
+            -------
+            stat : dict
+            """
+            raise NotImplementedError('FileSystem.stat')
+
+        def rm(self, path, recursive=False):
+            """
+            Alias for FileSystem.delete
+            """
+            return self.delete(path, recursive=recursive)
+
+        def mv(self, path, new_path):
+            """
+            Alias for FileSystem.rename
+            """
+            return self.rename(path, new_path)
+
+        def rename(self, path, new_path):
+            """
+            Rename file, like UNIX mv command
+
+            Parameters
+            ----------
+            path : string
+                Path to alter
+            new_path : string
+                Path to move to
+            """
+            raise NotImplementedError('FileSystem.rename')
+
+        def mkdir(self, path, create_parents=True):
+            raise NotImplementedError
+
+        def exists(self, path):
+            raise NotImplementedError
+
+        def isdir(self, path):
+            """
+            Return True if path is a directory
+            """
+            raise NotImplementedError
+
+        def isfile(self, path):
+            """
+            Return True if path is a file
+            """
+            raise NotImplementedError
+
+        def _isfilestore(self):
+            """
+            Returns True if this FileSystem is a unix-style file store with
+            directories.
+            """
+            raise NotImplementedError
+
+        def read_parquet(self, path, columns=None, metadata=None, schema=None,
+                         use_threads=True, use_pandas_metadata=False):
+            """
+            Read Parquet data from path in file system. Can read from a single file
+            or a directory of files
+
+            Parameters
+            ----------
+            path : str
+                Single file path or directory
+            columns : List[str], optional
+                Subset of columns to read
+            metadata : pyarrow.parquet.FileMetaData
+                Known metadata to validate files against
+            schema : pyarrow.parquet.Schema
+                Known schema to validate files against. Alternative to metadata
+                argument
+            use_threads : boolean, default True
+                Perform multi-threaded column reads
+            use_pandas_metadata : boolean, default False
+                If True and file has custom pandas schema metadata, ensure that
+                index columns are also loaded
+
+            Returns
+            -------
+            table : pyarrow.Table
+            """
+            from pyarrow.parquet import ParquetDataset
+            dataset = ParquetDataset(path, schema=schema, metadata=metadata,
+                                     filesystem=self)
+            return dataset.read(columns=columns, use_threads=use_threads,
+                                use_pandas_metadata=use_pandas_metadata)
+
+        def open(self, path, mode='rb'):
+            """
+            Open file for reading or writing
+            """
+            raise NotImplementedError
+
+        @property
+        def pathsep(self):
+            return '/'
+
+    class LocalFileSystem(FileSystem):
+
+        _instance = None
+
+        @classmethod
+        def get_instance(cls):
+            if cls._instance is None:
+                cls._instance = LocalFileSystem()
+            return cls._instance
+
+        @implements(FileSystem.ls)
+        def ls(self, path):
+            path = _stringify_path(path)
+            return sorted(pjoin(path, x) for x in os.listdir(path))
+
+        @implements(FileSystem.mkdir)
+        def mkdir(self, path, create_parents=True):
+            path = _stringify_path(path)
+            if create_parents:
+                os.makedirs(path)
+            else:
+                os.mkdir(path)
+
+        @implements(FileSystem.isdir)
+        def isdir(self, path):
+            path = _stringify_path(path)
+            return os.path.isdir(path)
+
+        @implements(FileSystem.isfile)
+        def isfile(self, path):
+            path = _stringify_path(path)
+            return os.path.isfile(path)
+
+        @implements(FileSystem._isfilestore)
+        def _isfilestore(self):
+            return True
+
+        @implements(FileSystem.exists)
+        def exists(self, path):
+            path = _stringify_path(path)
+            return os.path.exists(path)
+
+        @implements(FileSystem.open)
+        def open(self, path, mode='rb'):
+            """
+            Open file for reading or writing
+            """
+            path = _stringify_path(path)
+            return open(path, mode=mode)
+
+        @property
+        def pathsep(self):
+            return os.path.sep
+
+        def walk(self, path):
+            """
+            Directory tree generator, see os.walk
+            """
+            path = _stringify_path(path)
+            return os.walk(path)
+
+try:
+    from pyarrow import HadoopFileSystem
+except ImportError:  # pragma: no cover
+    HadoopFileSystem = object
+
 try:
     import lz4
     import lz4.frame
@@ -42,6 +296,7 @@
     compressions['lz4'] = lz4.frame.open
 
 FileSystem = FileSystem
+ArrowLocalFileSystem = LocalFileSystem
 
 
 class LocalFileSystem(ArrowLocalFileSystem):
@@ -109,6 +364,8 @@ def get_fs(path, storage_options):
     if scheme == 'file':
         return file_systems[scheme].get_instance()
     else:
+        if scheme == 'hdfs' and HadoopFileSystem is object:
+            raise ImportError('Need to install pyarrow to connect to HDFS.')
         options = _parse_from_path(path)
         storage_options = storage_options or dict()
         storage_options.update(options)


Test patch:
diff --git a/mars/tests/test_filesystem.py b/mars/tests/test_filesystem.py
--- a/mars/tests/test_filesystem.py
+++ b/mars/tests/test_filesystem.py
@@ -15,12 +15,12 @@
 import os
 import shutil
 import tempfile
+import unittest
 
-from mars.tests.core import TestBase
 from mars.filesystem import _parse_from_path, LocalFileSystem, glob, FSMap
 
 
-class Test(TestBase):
+class Test(unittest.TestCase):
 
     def testPathParser(self):
         path = 'hdfs://user:password@localhost:8080/test'