Repository: mars-project/mars

Issue:
[BUG] read parquet from a hdfs directory failed
**Describe the bug**
when `read_parquet` from a hdfs directory which contains some parquet files, exception occurs as it take the directory as a file by mistakes.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version: 3.7.9
2. The version of Mars you use: pymars[distributed]==0.6.0
3. Versions of crucial packages, such as numpy, scipy and protobuf: numpy==1.19.4, scipy==1.5.4, protobuf==3.14.0, pyarrow==2.0.0
4. Full stack of the error.
```
20/12/13 13:13:40 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
hdfsOpenFile(hdfs://<hdfs_ip>:8020/user/test/parquet_test): FileSystem#open((Lorg/apache/hadoop/fs/Path;I)Lorg/apache/hadoop/fs/FSDataInputStream;) error:
RemoteException: Path is not a file: /user/test/parquet_test
        at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:70)
        at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:56)
        at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsInt(FSNamesystem.java:2092)
        at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:2062)
        at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1975)
        at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:575)
        at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.getBlockLocations(AuthorizationProviderProxyClientProtocol.java:92)
        at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:376)
        at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
        at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617)
        at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1073)
        at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2226)
        at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2222)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:422)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1917)
        at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2220)
java.io.FileNotFoundException: Path is not a file: /user/test/parquet_test
        at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:70)
        at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:56)
        at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsInt(FSNamesystem.java:2092)
        at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:2062)
        at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1975)
        at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:575)
        at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.getBlockLocations(AuthorizationProviderProxyClientProtocol.java:92)
        at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:376)
        at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
        at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617)
        at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1073)
        at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2226)
        at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2222)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:422)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1917)
        at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2220)

        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
        at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
        at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:73)
        at org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:1289)
        at org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:1274)
        at org.apache.hadoop.hdfs.DFSClient.getLocatedBlocks(DFSClient.java:1262)
        at org.apache.hadoop.hdfs.DFSInputStream.fetchLocatedBlocksAndGetLastBlockLength(DFSInputStream.java:307)
        at org.apache.hadoop.hdfs.DFSInputStream.openInfo(DFSInputStream.java:273)
        at org.apache.hadoop.hdfs.DFSInputStream.<init>(DFSInputStream.java:265)
        at org.apache.hadoop.hdfs.DFSClient.open(DFSClient.java:1593)
        at org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:338)
        at org.apache.hadoop.hdfs.DistributedFileSystem$4.doCall(DistributedFileSystem.java:334)
        at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
        at org.apache.hadoop.hdfs.DistributedFileSystem.open(DistributedFileSystem.java:334)
Caused by: org.apache.hadoop.ipc.RemoteException(java.io.FileNotFoundException): Path is not a file: /user/test/parquet_test
        at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:70)
        at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:56)
        at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsInt(FSNamesystem.java:2092)
        at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:2062)
        at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1975)
        at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:575)
        at org.apache.hadoop.hdfs.server.namenode.AuthorizationProviderProxyClientProtocol.getBlockLocations(AuthorizationProviderProxyClientProtocol.java:92)
        at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:376)
        at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
        at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:617)
        at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1073)
        at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2226)
        at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2222)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:422)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1917)
        at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2220)

        at org.apache.hadoop.ipc.Client.call(Client.java:1504)
        at org.apache.hadoop.ipc.Client.call(Client.java:1441)
        at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230)
        at com.sun.proxy.$Proxy10.getBlockLocations(Unknown Source)
        at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getBlockLocations(ClientNamenodeProtocolTranslatorPB.java:266)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:260)
        at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104)
        at com.sun.proxy.$Proxy11.getBlockLocations(Unknown Source)
        at org.apache.hadoop.hdfs.DFSClient.callGetBlockLocations(DFSClient.java:1287)
        ... 10 more
Traceback (most recent call last):
  File "read_hdfs_dir.py", line 12, in <module>
    df = md.read_parquet('hdfs://<hdfs_ip>:8020/user/test/parquet_test')
  File "/home/test/lib/anaconda3/lib/python3.7/site-packages/mars/dataframe/datasource/read_parquet.py", line 394, in read_parquet
    with open_file(file_path, storage_options=storage_options) as f:
  File "/home/test/lib/anaconda3/lib/python3.7/site-packages/mars/filesystem.py", line 383, in open_file
    f = fs.open(path, mode=mode)
  File "pyarrow/io-hdfs.pxi", line 409, in pyarrow.lib.HadoopFileSystem.open
  File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status
OSError: HDFS path exists, but opening file failed: hdfs://<hdfs_ip>:8020/user/test/parquet_test
```
5. Minimized code to reproduce the error.
```python
import os

import mars.dataframe as md
from mars.session import new_session

os.environ['ARROW_LIBHDFS_DIR'] = '/opt/cloudera/parcels/CDH/lib64/'
new_session('http://<web_ip>:<web_port>').as_default()

df = md.read_parquet('hdfs://<hdfs_ip>:8020/user/test/parquet_test')
print(df.execute())
```

**Expected behavior**
read the directory correctly

**Additional context**
The hdfs directory strcture shown as below
```
test@myhost1 ~> hdfs dfs -du -h /user/test/parquet_test
136.0 M  272.0 M  /user/test/parquet_test/msg_0.parquet
136.0 M  272.0 M  /user/test/parquet_test/msg_1.parquet
136.0 M  272.0 M  /user/test/parquet_test/msg_2.parquet
136.0 M  272.0 M  /user/test/parquet_test/msg_3.parquet
136.0 M  272.0 M  /user/test/parquet_test/msg_4.parquet
136.0 M  272.0 M  /user/test/parquet_test/msg_5.parquet
136.0 M  272.0 M  /user/test/parquet_test/msg_6.parquet
136.0 M  272.0 M  /user/test/parquet_test/msg_7.parquet
136.0 M  272.0 M  /user/test/parquet_test/msg_8.parquet
136.0 M  272.0 M  /user/test/parquet_test/msg_9.parquet
```



Discussion:
Thank you for your report, we will address this issue ASAP.

Resolving patch:
diff --git a/mars/dataframe/datasource/read_csv.py b/mars/dataframe/datasource/read_csv.py
--- a/mars/dataframe/datasource/read_csv.py
+++ b/mars/dataframe/datasource/read_csv.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 from io import BytesIO
+from urllib.parse import urlparse
 
 import pandas as pd
 import numpy as np
@@ -24,7 +25,7 @@
 from ...core import OutputType
 from ...utils import parse_readable_size, lazy_import, FixedSizeFileObject
 from ...serialize import StringField, DictField, ListField, Int32Field, Int64Field, BoolField, AnyField
-from ...filesystem import open_file, file_size, glob
+from ...filesystem import get_fs, open_file, file_size, glob
 from ..arrays import ArrowStringDtype
 from ..core import IndexValue
 from ..utils import parse_index, build_empty_df, standardize_range_index, \
@@ -200,11 +201,23 @@ def tile(cls, op):
             # check if use_arrow_dtype set on the server side
             dtypes = to_arrow_dtypes(df.dtypes)
 
-        paths = op.path if isinstance(op.path, (tuple, list)) else glob(op.path, storage_options=op.storage_options)
+        path_prefix = ''
+        if isinstance(op.path, (tuple, list)):
+            paths = op.path
+        elif get_fs(op.path, op.storage_options).isdir(op.path):
+            parsed_path = urlparse(op.path)
+            if parsed_path.scheme.lower() == 'hdfs':
+                path_prefix = f'{parsed_path.scheme}://{parsed_path.netloc}'
+                paths = get_fs(op.path, op.storage_options).ls(op.path)
+            else:
+                paths = glob(op.path.rstrip('/') + '/*', storage_options=op.storage_options)
+        else:
+            paths = glob(op.path, storage_options=op.storage_options)
 
         out_chunks = []
         index_num = 0
         for path in paths:
+            path = path_prefix + path
             total_bytes = file_size(path)
             offset = 0
             for _ in range(int(np.ceil(total_bytes * 1.0 / chunk_bytes))):
@@ -616,8 +629,16 @@ def read_csv(path, names=None, sep=',', index_col=None, compression=None, header
     # infer dtypes and columns
     if isinstance(path, (list, tuple)):
         file_path = path[0]
+    elif get_fs(path, storage_options).isdir(path):
+        parsed_path = urlparse(path)
+        if parsed_path.scheme.lower() == 'hdfs':
+            path_prefix = f'{parsed_path.scheme}://{parsed_path.netloc}'
+            file_path = path_prefix + get_fs(path, storage_options).ls(path)[0]
+        else:
+            file_path = glob(path.rstrip('/') + '/*', storage_options)[0]
     else:
-        file_path = glob(path)[0]
+        file_path = glob(path, storage_options)[0]
+
     with open_file(file_path, compression=compression, storage_options=storage_options) as f:
         if head_lines is not None:
             b = b''.join([f.readline() for _ in range(head_lines)])
diff --git a/mars/dataframe/datasource/read_parquet.py b/mars/dataframe/datasource/read_parquet.py
--- a/mars/dataframe/datasource/read_parquet.py
+++ b/mars/dataframe/datasource/read_parquet.py
@@ -16,6 +16,7 @@
 
 import os
 import pickle
+from urllib.parse import urlparse
 
 import numpy as np
 import pandas as pd
@@ -33,7 +34,7 @@
 
 from ... import opcodes as OperandDef
 from ...config import options
-from ...filesystem import open_file, glob
+from ...filesystem import get_fs, open_file, glob
 from ...serialize import AnyField, BoolField, DictField, ListField,\
     StringField, Int32Field, BytesField
 from ..arrays import ArrowStringDtype
@@ -218,11 +219,17 @@ def _tile_partitioned(cls, op):
         dtypes = cls._to_arrow_dtypes(out_df.dtypes, op)
         dataset = pq.ParquetDataset(op.path)
 
+        parsed_path = urlparse(op.path)
+        if not os.path.exists(op.path) and parsed_path.scheme:
+            path_prefix = f'{parsed_path.scheme}://{parsed_path.netloc}'
+        else:
+            path_prefix = ''
+
         chunk_index = 0
         out_chunks = []
         for piece in dataset.pieces:
             chunk_op = op.copy().reset_key()
-            chunk_op._path = piece.path
+            chunk_op._path = path_prefix + piece.path
             chunk_op._partitions = pickle.dumps(dataset.partitions)
             chunk_op._partition_keys = piece.partition_keys
             new_chunk = chunk_op.new_chunk(
@@ -288,7 +295,7 @@ def _tile_no_partitioned(cls, op):
 
     @classmethod
     def tile(cls, op):
-        if os.path.isdir(op.path):
+        if get_fs(op.path, op.storage_options).isdir(op.path):
             return cls._tile_partitioned(op)
         else:
             return cls._tile_no_partitioned(op)
@@ -376,7 +383,7 @@ def read_parquet(path, engine: str = "auto", columns=None,
     engine_type = check_engine(engine)
     engine = get_engine(engine_type)
 
-    if os.path.isdir(path):
+    if get_fs(path, storage_options).isdir(path):
         # If path is a directory, we will read as a partitioned datasets.
         if engine_type != 'pyarrow':
             raise TypeError('Only support pyarrow engine when reading from'


Test patch:
diff --git a/mars/dataframe/datasource/tests/test_datasource_execution.py b/mars/dataframe/datasource/tests/test_datasource_execution.py
--- a/mars/dataframe/datasource/tests/test_datasource_execution.py
+++ b/mars/dataframe/datasource/tests/test_datasource_execution.py
@@ -478,6 +478,28 @@ def testReadCSVExecution(self):
                 md.read_csv(f'{tempdir}/*.csv', index_col=0, chunk_bytes=50), concat=True)[0]
             pd.testing.assert_frame_equal(df, mdf2.sort_index())
 
+        # test read directory
+        with tempfile.TemporaryDirectory() as tempdir:
+            testdir = os.path.join(tempdir, 'test_dir')
+            os.makedirs(testdir, exist_ok=True)
+
+            df = pd.DataFrame(np.random.rand(300, 3), columns=['a', 'b', 'c'])
+
+            file_paths = [os.path.join(testdir, f'test{i}.csv') for i in range(3)]
+            df[:100].to_csv(file_paths[0])
+            df[100:200].to_csv(file_paths[1])
+            df[200:].to_csv(file_paths[2])
+
+            # As we can not guarantee the order in which these files are processed,
+            # the result may not keep the original order.
+            mdf = self.executor.execute_dataframe(
+                md.read_csv(testdir, index_col=0), concat=True)[0]
+            pd.testing.assert_frame_equal(df, mdf.sort_index())
+
+            mdf2 = self.executor.execute_dataframe(
+                md.read_csv(testdir, index_col=0, chunk_bytes=50), concat=True)[0]
+            pd.testing.assert_frame_equal(df, mdf2.sort_index())
+
     @unittest.skipIf(pa is None, 'pyarrow not installed')
     def testReadCSVUseArrowDtype(self):
         df = pd.DataFrame({
@@ -765,11 +787,11 @@ def testDateRangeExecution(self):
         expected = pd.date_range(start='1/1/2018', periods=5, freq='M')
         pd.testing.assert_index_equal(result, expected)
 
-    @unittest.skipIf(pa is None or fastparquet is None, 'pyarrow or fastparquet not installed')
-    def testReadParquet(self):
+    @unittest.skipIf(pa is None, 'pyarrow not installed')
+    def testReadParquetArrow(self):
         test_df = pd.DataFrame({'a': np.arange(10).astype(np.int64, copy=False),
                                 'b': [f's{i}' for i in range(10)],
-                                'c': np.random.rand(10),})
+                                'c': np.random.rand(10), })
 
         with tempfile.TemporaryDirectory() as tempdir:
             file_path = os.path.join(tempdir, 'test.csv')
@@ -799,15 +821,6 @@ def testReadParquet(self):
             self.assertIsInstance(result.dtypes.iloc[1], md.ArrowStringDtype)
             pd.testing.assert_frame_equal(arrow_array_to_objects(result), test_df)
 
-        # test fastparquet engine
-        with tempfile.TemporaryDirectory() as tempdir:
-            file_path = os.path.join(tempdir, 'test.csv')
-            test_df.to_parquet(file_path, compression=None)
-
-            df = md.read_parquet(file_path, engine='fastparquet')
-            result = self.executor.execute_dataframe(df, concat=True)[0]
-            pd.testing.assert_frame_equal(result, test_df)
-
         # test wildcards in path
         with tempfile.TemporaryDirectory() as tempdir:
             df = pd.DataFrame({'a': np.arange(300).astype(np.int64, copy=False),
@@ -826,3 +839,18 @@ def testReadParquet(self):
             mdf = md.read_parquet(f'{tempdir}/*.parquet', groups_as_chunks=True)
             r = self.executor.execute_dataframe(mdf, concat=True)[0]
             pd.testing.assert_frame_equal(df, r.sort_values('a').reset_index(drop=True))
+
+    @unittest.skipIf(fastparquet is None, 'fastparquet not installed')
+    def testReadParquetFastParquet(self):
+        test_df = pd.DataFrame({'a': np.arange(10).astype(np.int64, copy=False),
+                                'b': [f's{i}' for i in range(10)],
+                                'c': np.random.rand(10), })
+
+        # test fastparquet engine
+        with tempfile.TemporaryDirectory() as tempdir:
+            file_path = os.path.join(tempdir, 'test.csv')
+            test_df.to_parquet(file_path, compression=None)
+
+            df = md.read_parquet(file_path, engine='fastparquet')
+            result = self.executor.execute_dataframe(df, concat=True)[0]
+            pd.testing.assert_frame_equal(result, test_df)
diff --git a/mars/dataframe/datasource/tests/test_hdfs.py b/mars/dataframe/datasource/tests/test_hdfs.py
--- a/mars/dataframe/datasource/tests/test_hdfs.py
+++ b/mars/dataframe/datasource/tests/test_hdfs.py
@@ -12,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import unittest
 import os
-from io import BytesIO
+import random
+import unittest
+from io import BytesIO, StringIO
 
 import pandas as pd
 import numpy as np
@@ -25,20 +26,6 @@
 
 TEST_DIR = '/tmp/test'
 
-csv_content = b"""
-A,B,C,D,E
-1.0,2020-01-01,1.0,3,foo
-1.0,2020-01-01,1.0,3,foo
-1.0,2020-01-01,1.0,3,foo
-1.0,2020-01-01,1.0,3,foo
-1.0,2020-01-01,1.0,3,foo
-1.0,2020-01-01,1.0,3,foo
-1.0,2020-01-01,1.0,3,foo
-1.0,2020-01-01,1.0,3,foo
-1.0,2020-01-01,1.0,3,foo
-1.0,2020-01-01,1.0,3,foo
-""".strip()
-
 
 @unittest.skipIf(not os.environ.get('WITH_HADOOP'), 'Only run when hdfs is installed')
 class TestHDFS(TestBase):
@@ -63,6 +50,22 @@ def testReadCSVExecution(self):
         res = df.to_pandas()
         pd.testing.assert_frame_equal(expected, res)
 
+        test_df = pd.DataFrame({
+            'A': np.random.rand(20),
+            'B': [pd.Timestamp('2020-01-01') + pd.Timedelta(days=random.randint(0, 31))
+                  for _ in range(20)],
+            'C': np.random.rand(20),
+            'D': np.random.randint(0, 100, size=(20,)),
+            'E': ['foo' + str(random.randint(0, 999999)) for _ in range(20)],
+        })
+        buf = StringIO()
+        test_df[:10].to_csv(buf)
+        csv_content = buf.getvalue().encode()
+
+        buf = StringIO()
+        test_df[10:].to_csv(buf)
+        csv_content2 = buf.getvalue().encode()
+
         with self.hdfs.open(f"{TEST_DIR}/chunk_test.csv", "wb", replication=1) as f:
             f.write(csv_content)
 
@@ -71,13 +74,40 @@ def testReadCSVExecution(self):
         res = df.to_pandas()
         pd.testing.assert_frame_equal(expected.reset_index(drop=True), res.reset_index(drop=True))
 
+        test_read_dir = f'{TEST_DIR}/test_read_csv_directory'
+        self.hdfs.mkdir(test_read_dir)
+        with self.hdfs.open(f"{test_read_dir}/part.csv", "wb", replication=1) as f:
+            f.write(csv_content)
+        with self.hdfs.open(f"{test_read_dir}/part2.csv", "wb", replication=1) as f:
+            f.write(csv_content2)
+
+        df = md.read_csv(f'hdfs://localhost:8020{test_read_dir}', chunk_bytes=50)
+        expected = pd.concat([pd.read_csv(BytesIO(csv_content)), pd.read_csv(BytesIO(csv_content2))])
+        res = df.to_pandas()
+        pd.testing.assert_frame_equal(expected.reset_index(drop=True), res.reset_index(drop=True))
+
     def testReadParquetExecution(self):
         test_df = pd.DataFrame({'a': np.arange(10).astype(np.int64, copy=False),
                                 'b': [f's{i}' for i in range(10)],
                                 'c': np.random.rand(10), })
+        test_df2 = pd.DataFrame({'a': np.arange(10).astype(np.int64, copy=False),
+                                 'b': [f's{i}' for i in range(10)],
+                                 'c': np.random.rand(10), })
+
         with self.hdfs.open(f"{TEST_DIR}/test.parquet", "wb", replication=1) as f:
             test_df.to_parquet(f, row_group_size=3)
 
         df = md.read_parquet(f'hdfs://localhost:8020{TEST_DIR}/test.parquet')
         res = df.to_pandas()
         pd.testing.assert_frame_equal(res, test_df)
+
+        self.hdfs.mkdir(f"{TEST_DIR}/test_partitioned")
+
+        with self.hdfs.open(f"{TEST_DIR}/test_partitioned/file1.parquet", "wb", replication=1) as f:
+            test_df.to_parquet(f, row_group_size=3)
+        with self.hdfs.open(f"{TEST_DIR}/test_partitioned/file2.parquet", "wb", replication=1) as f:
+            test_df2.to_parquet(f, row_group_size=3)
+
+        df = md.read_parquet(f'hdfs://localhost:8020{TEST_DIR}/test_partitioned')
+        res = df.to_pandas()
+        pd.testing.assert_frame_equal(res, pd.concat([test_df, test_df2]))
diff --git a/mars/dataframe/datastore/tests/test_datastore_execute.py b/mars/dataframe/datastore/tests/test_datastore_execute.py
--- a/mars/dataframe/datastore/tests/test_datastore_execute.py
+++ b/mars/dataframe/datastore/tests/test_datastore_execute.py
@@ -172,8 +172,8 @@ def testWithGivenSession(session):
             with new_session('http://' + cluster._web_endpoint).as_default() as web_session:
                 testWithGivenSession(web_session)
 
-    @unittest.skipIf(pa is None or fastparquet is None, 'pyarrow or fastparquet not installed')
-    def testToParquetExecution(self):
+    @unittest.skipIf(pa is None, 'pyarrow not installed')
+    def testToParquetArrowExecution(self):
         raw = pd.DataFrame({
             'col1': np.random.rand(100),
             'col2': np.arange(100),
@@ -192,11 +192,6 @@ def testToParquetExecution(self):
             result = result.sort_index()
             pd.testing.assert_frame_equal(result, raw)
 
-            # test fastparquet
-            path = os.path.join(base_path, 'out-fastparquet-*.parquet')
-            r = df.to_parquet(path, engine='fastparquet', compression='gzip')
-            self.executor.execute_dataframe(r)
-
             read_df = md.read_parquet(path)
             result = self.executor.execute_dataframe(read_df, concat=True)[0]
             result = result.sort_index()
@@ -212,3 +207,18 @@ def testToParquetExecution(self):
             result['col3'] = result['col3'].astype('object')
             pd.testing.assert_frame_equal(result.sort_values('col1').reset_index(drop=True),
                                           raw.sort_values('col1').reset_index(drop=True))
+
+    @unittest.skipIf(fastparquet is None, 'fastparquet not installed')
+    def testToParquetFastParquetExecution(self):
+        raw = pd.DataFrame({
+            'col1': np.random.rand(100),
+            'col2': np.arange(100),
+            'col3': np.random.choice(['a', 'b', 'c'], (100,)),
+        })
+        df = DataFrame(raw, chunk_size=33)
+
+        with tempfile.TemporaryDirectory() as base_path:
+            # test fastparquet
+            path = os.path.join(base_path, 'out-fastparquet-*.parquet')
+            r = df.to_parquet(path, engine='fastparquet', compression='gzip')
+            self.executor.execute_dataframe(r)