Repository: dask/dask

Issue:
Update read_csv, read_hdf, and read_table to take a list of files
I have found that 

    dd.read_csv('myfiles.*.csv')
    dd.read_hdf('myfiles.*.csv', key='whatever')

could be improved by allowing an iterable of filenames to be passed in as the `urlpath` argument.  This allows more fine tuned filtering than the glob patterns and allows you to sample the `urlpath` during development.

    # example usage
    files = glob('myfiles.*.csv')
    files = random.sample(files, 5)  # work on a subset of the files for initial development
    df = dd.read_csv(files)

I also want `read_hdf` and `read_table` to have the same feature.  

My current workaround is to

    # example usage
    files = glob('myfiles.*.csv')
    files = random.sample(files, 5)  # work on a subset of the files for initial development
    dfs = [dd.read_csv(f) for f in files]
    df = dd.concat(dfs)




Discussion:
Related issue.
Is there a method to get the list of files discovered from the glob pattern?
While we are wishing, I'd like to see REGEX support for the urlpath.
Thanks.
Sorry for not having noticed this before, @jtf621 - it is certainly something we can implement.

@apiszcz : there is currently no way to extract the list of files that were used, but you can look at the file-system classes dask.bytes.core._filesystems, e.g., dask.bytes.local.LocalFileSystem, and call it's glob method (which in this case called the builtin glob function).

For regex support, would you imagine passing `regex=True` to evaluate the path as a matching pattern?
Thank you, That would be great, another capability I'm in search of is to
obtain a list/dictionary of the input filenames to chunks or partitions.
regex=True is fine, glob is limited compared to regex.

On Fri, Jul 14, 2017 at 9:30 AM, Martin Durant <notifications@github.com>
wrote:

> Sorry for not having noticed this before, @jtf621
> <https://github.com/jtf621> - it is certainly something we can implement.
>
> @apiszcz <https://github.com/apiszcz> : there is currently no way to
> extract the list of files that were used, but you can look at the
> file-system classes dask.bytes.core._filesystems, e.g., dask.bytes.local.LocalFileSystem,
> and call it's glob method (which in this case called the builtin glob
> function).
>
> For regex support, would you imagine passing regex=True to evaluate the
> path as a matching pattern?
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/1948#issuecomment-315359839>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/ABXVTS4bHPNgnbcTYw1fJk9_R43OJRpBks5sN21wgaJpZM4LyEjU>
> .
>

"obtain a list/dictionary of the input filenames to chunks or partitions" - I could see how this information might look, but I don't see how it could be returned. The purpose of the functions is to create dataframes, so it would not be appropriate to return anything else.  It would be possible to add a column to each partition with the data's origin, or put the file/offset in the partition's `name` field, but this would take a bit of untangling the current code path.
Note that read_csv already does support passing a list of files.
and a name filter, thanks.

On Fri, Jul 14, 2017 at 3:58 PM, Martin Durant <notifications@github.com>
wrote:

> Note that read_csv already does support passing a list of files.
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/1948#issuecomment-315453121>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/ABXVTefEvidXdgR_yigQ_QgP_VnPh53Pks5sN8hVgaJpZM4LyEjU>
> .
>

"name filter" - sorry, what do you mean here? I thought you wanted to pass your own list of filenames or a regex to match against.

Resolving patch:
diff --git a/dask/bag/text.py b/dask/bag/text.py
--- a/dask/bag/text.py
+++ b/dask/bag/text.py
@@ -21,9 +21,11 @@ def read_text(urlpath, blocksize=None, compression='infer',
 
     Parameters
     ----------
-    urlpath: string or list
-        Absolute or relative filepath, URL (may include protocols like
-        ``s3://``), globstring, or a list of beforementioned strings.
+    urlpath : string or list
+        Absolute or relative filepath(s). Prefix with a protocol like ``s3://``
+        to read from alternative filesystems. To read from multiple files you
+        can pass a globstring or a list of paths, with the caveat that they
+        must all have the same protocol.
     blocksize: None or int
         Size (in bytes) to cut up larger files.  Streams by default.
     compression: string
@@ -59,29 +61,18 @@ def read_text(urlpath, blocksize=None, compression='infer',
     --------
     from_sequence: Build bag from Python sequence
     """
-    if isinstance(urlpath, (tuple, list, set)):
-        blocks = sum([read_text(fn, blocksize=blocksize,
-                      compression=compression, encoding=encoding, errors=errors,
-                      linedelimiter=linedelimiter, collection=False,
-                      storage_options=storage_options)
-                     for fn in urlpath], [])
+    if blocksize is None:
+        files = open_files(urlpath, mode='rt', encoding=encoding,
+                           errors=errors, compression=compression,
+                           **(storage_options or {}))
+        blocks = [delayed(list)(delayed(file_to_blocks)(fil)) for fil in files]
+
     else:
-        if blocksize is None:
-            files = open_files(urlpath, mode='rt', encoding=encoding,
-                               errors=errors, compression=compression,
+        _, blocks = read_bytes(urlpath, delimiter=linedelimiter.encode(),
+                               blocksize=blocksize, sample=False,
+                               compression=compression,
                                **(storage_options or {}))
-            blocks = [delayed(list, pure=True)(delayed(file_to_blocks)(file))
-                      for file in files]
-
-        else:
-            _, blocks = read_bytes(urlpath, delimiter=linedelimiter.encode(),
-                                   blocksize=blocksize, sample=False,
-                                   compression=compression,
-                                   **(storage_options or {}))
-            if isinstance(blocks[0], (tuple, list)):
-                blocks = list(concat(blocks))
-            blocks = [delayed(decode)(b, encoding, errors)
-                      for b in blocks]
+        blocks = [delayed(decode)(b, encoding, errors) for b in concat(blocks)]
 
     if not blocks:
         raise ValueError("No files found", urlpath)
diff --git a/dask/dataframe/io/csv.py b/dask/dataframe/io/csv.py
--- a/dask/dataframe/io/csv.py
+++ b/dask/dataframe/io/csv.py
@@ -366,9 +366,11 @@ def read_pandas(reader, urlpath, blocksize=AUTO_BLOCKSIZE, collection=True,
 
 Parameters
 ----------
-urlpath : string
-    Absolute or relative filepath, URL (may include protocols like
-    ``s3://``), or globstring for {file_type} files.
+urlpath : string or list
+    Absolute or relative filepath(s). Prefix with a protocol like ``s3://``
+    to read from alternative filesystems. To read from multiple files you
+    can pass a globstring or a list of paths, with the caveat that they
+    must all have the same protocol.
 blocksize : int or None, optional
     Number of bytes by which to cut up larger files. Default value is
     computed based on available physical memory and the number of cores.
diff --git a/dask/dataframe/io/parquet.py b/dask/dataframe/io/parquet.py
--- a/dask/dataframe/io/parquet.py
+++ b/dask/dataframe/io/parquet.py
@@ -718,9 +718,12 @@ def read_parquet(path, columns=None, filters=None, categories=None, index=None,
 
     Parameters
     ----------
-    path : string
-        Source directory for data. May be a glob string.
-        Prepend with protocol like ``s3://`` or ``hdfs://`` for remote data.
+    path : string or list
+        Source directory for data, or path(s) to individual parquet files.
+        Prefix with a protocol like ``s3://`` to read from alternative
+        filesystems. To read from multiple files you can pass a globstring or a
+        list of paths, with the caveat that they must all have the same
+        protocol.
     columns: list or None
         List of column names to load
     filters: list


Test patch:
diff --git a/dask/bag/tests/test_text.py b/dask/bag/tests/test_text.py
--- a/dask/bag/tests/test_text.py
+++ b/dask/bag/tests/test_text.py
@@ -41,6 +41,11 @@ def test_read_text(fmt, bs, encoding):
         L, = compute(b)
         assert ''.join(L) == expected
 
+        b = read_text(sorted(files), compression=fmt, blocksize=bs,
+                      encoding=encoding)
+        L, = compute(b)
+        assert ''.join(L) == expected
+
         blocks = read_text('.test.accounts.*.json', compression=fmt, blocksize=bs,
                            encoding=encoding, collection=False)
         L = compute(*blocks)
diff --git a/dask/dataframe/io/tests/test_csv.py b/dask/dataframe/io/tests/test_csv.py
--- a/dask/dataframe/io/tests/test_csv.py
+++ b/dask/dataframe/io/tests/test_csv.py
@@ -270,6 +270,20 @@ def test_read_csv_files(dd_read, pd_read, files):
         assert_eq(df, expected2, check_dtype=False)
 
 
+@pytest.mark.parametrize('dd_read,pd_read,files',
+                         [(dd.read_csv, pd.read_csv, csv_files),
+                          (dd.read_table, pd.read_table, tsv_files)])
+def test_read_csv_files_list(dd_read, pd_read, files):
+    with filetexts(files, mode='b'):
+        subset = sorted(files)[:2]  # Just first 2
+        sol = pd.concat([pd_read(BytesIO(files[k])) for k in subset])
+        res = dd_read(subset)
+        assert_eq(res, sol, check_dtype=False)
+
+        with pytest.raises(ValueError):
+            dd_read([])
+
+
 # After this point, we test just using read_csv, as all functionality
 # for both is implemented using the same code.
 
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -151,6 +151,17 @@ def test_read_glob(tmpdir, write_engine, read_engine):
     assert_eq(df, ddf2)
 
 
+@write_read_engines()
+def test_read_list(tmpdir, write_engine, read_engine):
+    tmpdir = str(tmpdir)
+    ddf.to_parquet(tmpdir, engine=write_engine)
+    files = sorted(os.path.join(tmpdir, f)
+                   for f in os.listdir(tmpdir)
+                   if not f.endswith('_metadata'))
+    ddf2 = dd.read_parquet(files, engine=read_engine)
+    assert_eq(df, ddf2)
+
+
 @write_read_engines_xfail
 def test_auto_add_index(tmpdir, write_engine, read_engine):
     fn = str(tmpdir)