Repository: dask/dask

Issue:
[BUG?] Not able to filter and load a partitioned parquet file using dask
I'm having a problem in filtering and loading a partitioned parquet file using dask, which I show below.

*I'm including full code for reproducibility.*

# Versions

dask==0.17.0
pandas==0.22.0
fastparquet==0.1.4

<details>
<summary>Init code</summary>

# Imports


```python
import numpy as np
import pandas as pd
import dask.dataframe as dd
import random
import fastparquet
```

# Generation and plotting functions


```python
#generate dummy df
def generate_df(size):
    d = {
        'dates':
        pd.date_range('1980-01-01', periods=size, freq='1T'),
        'signal1':
        np.random.normal(0, 0.3, size=size).cumsum() + 50,
        'signal2':
        np.random.normal(0, 0.3, size=size).cumsum() + 50,
        'signal3':
        np.random.normal(0, 0.3, size=size).cumsum() + 50,
        'fake_categorical1':
        np.random.choice(['A', 'B', 'C'], size=size),
        'fake_categorical2':
        np.random.choice(['D', 'E', 'F'], size=size)
    }

    df = pd.DataFrame(d)

    return df.melt(id_vars=['dates', 'fake_categorical1', 'fake_categorical2'])
```

# Generate df

Let's generate a small df:


```python
small_df = generate_df(10)
small_df
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>dates</th>
      <th>fake_categorical1</th>
      <th>fake_categorical2</th>
      <th>variable</th>
      <th>value</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>1980-01-01 00:00:00</td>
      <td>C</td>
      <td>D</td>
      <td>signal1</td>
      <td>50.106769</td>
    </tr>
    <tr>
      <th>1</th>
      <td>1980-01-01 00:01:00</td>
      <td>C</td>
      <td>F</td>
      <td>signal1</td>
      <td>50.134314</td>
    </tr>
    <tr>
      <th>2</th>
      <td>1980-01-01 00:02:00</td>
      <td>B</td>
      <td>F</td>
      <td>signal1</td>
      <td>50.049454</td>
    </tr>
    <tr>
      <th>3</th>
      <td>1980-01-01 00:03:00</td>
      <td>C</td>
      <td>E</td>
      <td>signal1</td>
      <td>50.204364</td>
    </tr>
    <tr>
      <th>4</th>
      <td>1980-01-01 00:04:00</td>
      <td>A</td>
      <td>D</td>
      <td>signal1</td>
      <td>50.337799</td>
    </tr>
    <tr>
      <th>5</th>
      <td>1980-01-01 00:05:00</td>
      <td>A</td>
      <td>D</td>
      <td>signal1</td>
      <td>50.716083</td>
    </tr>
    <tr>
      <th>6</th>
      <td>1980-01-01 00:06:00</td>
      <td>B</td>
      <td>E</td>
      <td>signal1</td>
      <td>50.389717</td>
    </tr>
    <tr>
      <th>7</th>
      <td>1980-01-01 00:07:00</td>
      <td>C</td>
      <td>E</td>
      <td>signal1</td>
      <td>50.295345</td>
    </tr>
    <tr>
      <th>8</th>
      <td>1980-01-01 00:08:00</td>
      <td>C</td>
      <td>E</td>
      <td>signal1</td>
      <td>50.221086</td>
    </tr>
    <tr>
      <th>9</th>
      <td>1980-01-01 00:09:00</td>
      <td>C</td>
      <td>E</td>
      <td>signal1</td>
      <td>50.074217</td>
    </tr>
    <tr>
      <th>10</th>
      <td>1980-01-01 00:00:00</td>
      <td>C</td>
      <td>D</td>
      <td>signal2</td>
      <td>50.475801</td>
    </tr>
    <tr>
      <th>11</th>
      <td>1980-01-01 00:01:00</td>
      <td>C</td>
      <td>F</td>
      <td>signal2</td>
      <td>49.874271</td>
    </tr>
    <tr>
      <th>12</th>
      <td>1980-01-01 00:02:00</td>
      <td>B</td>
      <td>F</td>
      <td>signal2</td>
      <td>49.745380</td>
    </tr>
    <tr>
      <th>13</th>
      <td>1980-01-01 00:03:00</td>
      <td>C</td>
      <td>E</td>
      <td>signal2</td>
      <td>49.967490</td>
    </tr>
    <tr>
      <th>14</th>
      <td>1980-01-01 00:04:00</td>
      <td>A</td>
      <td>D</td>
      <td>signal2</td>
      <td>49.822340</td>
    </tr>
    <tr>
      <th>15</th>
      <td>1980-01-01 00:05:00</td>
      <td>A</td>
      <td>D</td>
      <td>signal2</td>
      <td>49.740567</td>
    </tr>
    <tr>
      <th>16</th>
      <td>1980-01-01 00:06:00</td>
      <td>B</td>
      <td>E</td>
      <td>signal2</td>
      <td>49.393413</td>
    </tr>
    <tr>
      <th>17</th>
      <td>1980-01-01 00:07:00</td>
      <td>C</td>
      <td>E</td>
      <td>signal2</td>
      <td>49.151365</td>
    </tr>
    <tr>
      <th>18</th>
      <td>1980-01-01 00:08:00</td>
      <td>C</td>
      <td>E</td>
      <td>signal2</td>
      <td>49.381807</td>
    </tr>
    <tr>
      <th>19</th>
      <td>1980-01-01 00:09:00</td>
      <td>C</td>
      <td>E</td>
      <td>signal2</td>
      <td>48.647324</td>
    </tr>
    <tr>
      <th>20</th>
      <td>1980-01-01 00:00:00</td>
      <td>C</td>
      <td>D</td>
      <td>signal3</td>
      <td>49.837980</td>
    </tr>
    <tr>
      <th>21</th>
      <td>1980-01-01 00:01:00</td>
      <td>C</td>
      <td>F</td>
      <td>signal3</td>
      <td>49.787703</td>
    </tr>
    <tr>
      <th>22</th>
      <td>1980-01-01 00:02:00</td>
      <td>B</td>
      <td>F</td>
      <td>signal3</td>
      <td>49.616496</td>
    </tr>
    <tr>
      <th>23</th>
      <td>1980-01-01 00:03:00</td>
      <td>C</td>
      <td>E</td>
      <td>signal3</td>
      <td>49.674050</td>
    </tr>
    <tr>
      <th>24</th>
      <td>1980-01-01 00:04:00</td>
      <td>A</td>
      <td>D</td>
      <td>signal3</td>
      <td>49.987540</td>
    </tr>
    <tr>
      <th>25</th>
      <td>1980-01-01 00:05:00</td>
      <td>A</td>
      <td>D</td>
      <td>signal3</td>
      <td>49.667552</td>
    </tr>
    <tr>
      <th>26</th>
      <td>1980-01-01 00:06:00</td>
      <td>B</td>
      <td>E</td>
      <td>signal3</td>
      <td>49.553683</td>
    </tr>
    <tr>
      <th>27</th>
      <td>1980-01-01 00:07:00</td>
      <td>C</td>
      <td>E</td>
      <td>signal3</td>
      <td>50.074661</td>
    </tr>
    <tr>
      <th>28</th>
      <td>1980-01-01 00:08:00</td>
      <td>C</td>
      <td>E</td>
      <td>signal3</td>
      <td>50.526340</td>
    </tr>
    <tr>
      <th>29</th>
      <td>1980-01-01 00:09:00</td>
      <td>C</td>
      <td>E</td>
      <td>signal3</td>
      <td>51.246639</td>
    </tr>
  </tbody>
</table>
</div>

</details>

<details>
<summary>Here is where the problem shows</summary>

# Write and read non partitioned parquet:

Save a **non partitioned** parquet:


```python
fastparquet.write('file_non_partitioned', small_df, compression='snappy',
                  file_scheme='hive', write_index=False)
```

Read it with dask:


```python
df_non_partitioned = dd.read_parquet('file_non_partitioned')
```

No problems showing head():


```python
df_non_partitioned.head()
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>dates</th>
      <th>fake_categorical1</th>
      <th>fake_categorical2</th>
      <th>variable</th>
      <th>value</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>1980-01-01 00:00:00</td>
      <td>C</td>
      <td>D</td>
      <td>signal1</td>
      <td>50.106769</td>
    </tr>
    <tr>
      <th>1</th>
      <td>1980-01-01 00:01:00</td>
      <td>C</td>
      <td>F</td>
      <td>signal1</td>
      <td>50.134314</td>
    </tr>
    <tr>
      <th>2</th>
      <td>1980-01-01 00:02:00</td>
      <td>B</td>
      <td>F</td>
      <td>signal1</td>
      <td>50.049454</td>
    </tr>
    <tr>
      <th>3</th>
      <td>1980-01-01 00:03:00</td>
      <td>C</td>
      <td>E</td>
      <td>signal1</td>
      <td>50.204364</td>
    </tr>
    <tr>
      <th>4</th>
      <td>1980-01-01 00:04:00</td>
      <td>A</td>
      <td>D</td>
      <td>signal1</td>
      <td>50.337799</td>
    </tr>
  </tbody>
</table>
</div>



And no problems filtering on a value of a column and computing... cool! The problem comes with partitioned parquet (see below).


```python
df_non_partitioned[df_non_partitioned.fake_categorical1=='A'].compute()
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>dates</th>
      <th>fake_categorical1</th>
      <th>fake_categorical2</th>
      <th>variable</th>
      <th>value</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>4</th>
      <td>1980-01-01 00:04:00</td>
      <td>A</td>
      <td>D</td>
      <td>signal1</td>
      <td>50.337799</td>
    </tr>
    <tr>
      <th>5</th>
      <td>1980-01-01 00:05:00</td>
      <td>A</td>
      <td>D</td>
      <td>signal1</td>
      <td>50.716083</td>
    </tr>
    <tr>
      <th>14</th>
      <td>1980-01-01 00:04:00</td>
      <td>A</td>
      <td>D</td>
      <td>signal2</td>
      <td>49.822340</td>
    </tr>
    <tr>
      <th>15</th>
      <td>1980-01-01 00:05:00</td>
      <td>A</td>
      <td>D</td>
      <td>signal2</td>
      <td>49.740567</td>
    </tr>
    <tr>
      <th>24</th>
      <td>1980-01-01 00:04:00</td>
      <td>A</td>
      <td>D</td>
      <td>signal3</td>
      <td>49.987540</td>
    </tr>
    <tr>
      <th>25</th>
      <td>1980-01-01 00:05:00</td>
      <td>A</td>
      <td>D</td>
      <td>signal3</td>
      <td>49.667552</td>
    </tr>
  </tbody>
</table>
</div>




# Write and read partitioned parquet

Save a **partitioned** parquet:


```python
fastparquet.write('file_partitioned', small_df, compression='snappy',
                  file_scheme='hive', write_index=False,
                  partition_on=['fake_categorical1','fake_categorical2'])
```

Read it with dask:


```python
df_partitioned = dd.read_parquet('file_partitioned')
```

No problems showing head():


```python
df_partitioned.head()
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>dates</th>
      <th>variable</th>
      <th>value</th>
      <th>fake_categorical1</th>
      <th>fake_categorical2</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>1980-01-01 00:04:00</td>
      <td>signal1</td>
      <td>50.337799</td>
      <td>A</td>
      <td>D</td>
    </tr>
    <tr>
      <th>1</th>
      <td>1980-01-01 00:05:00</td>
      <td>signal1</td>
      <td>50.716083</td>
      <td>A</td>
      <td>D</td>
    </tr>
    <tr>
      <th>2</th>
      <td>1980-01-01 00:04:00</td>
      <td>signal2</td>
      <td>49.822340</td>
      <td>A</td>
      <td>D</td>
    </tr>
    <tr>
      <th>3</th>
      <td>1980-01-01 00:05:00</td>
      <td>signal2</td>
      <td>49.740567</td>
      <td>A</td>
      <td>D</td>
    </tr>
    <tr>
      <th>4</th>
      <td>1980-01-01 00:04:00</td>
      <td>signal3</td>
      <td>49.987540</td>
      <td>A</td>
      <td>D</td>
    </tr>
  </tbody>
</table>
</div>



No problems running `compute()` on the full dataframe:


```python
df_partitioned.compute().head()
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>dates</th>
      <th>variable</th>
      <th>value</th>
      <th>fake_categorical1</th>
      <th>fake_categorical2</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>1980-01-01 00:04:00</td>
      <td>signal1</td>
      <td>50.337799</td>
      <td>A</td>
      <td>D</td>
    </tr>
    <tr>
      <th>1</th>
      <td>1980-01-01 00:05:00</td>
      <td>signal1</td>
      <td>50.716083</td>
      <td>A</td>
      <td>D</td>
    </tr>
    <tr>
      <th>2</th>
      <td>1980-01-01 00:04:00</td>
      <td>signal2</td>
      <td>49.822340</td>
      <td>A</td>
      <td>D</td>
    </tr>
    <tr>
      <th>3</th>
      <td>1980-01-01 00:05:00</td>
      <td>signal2</td>
      <td>49.740567</td>
      <td>A</td>
      <td>D</td>
    </tr>
    <tr>
      <th>4</th>
      <td>1980-01-01 00:04:00</td>
      <td>signal3</td>
      <td>49.987540</td>
      <td>A</td>
      <td>D</td>
    </tr>
  </tbody>
</table>
</div>



But when trying to filtering on a column before computing it doesnt work:


```python
df_partitioned[df_partitioned.fake_categorical1=='A'].compute()
```

</details>

<details>
<summary>Error and more examples</summary>

    ---------------------------------------------------------------------------

    ValueError                                Traceback (most recent call last)

    <ipython-input-33-3a07acb2728f> in <module>()
    ----> 1 df_partitioned[df_partitioned.fake_categorical1=='A'].compute()
    

    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/base.py in compute(self, **kwargs)
        141         dask.base.compute
        142         """
    --> 143         (result,) = compute(self, traverse=False, **kwargs)
        144         return result
        145 


    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/base.py in compute(*args, **kwargs)
        390     postcomputes = [a.__dask_postcompute__() if is_dask_collection(a)
        391                     else (None, a) for a in args]
    --> 392     results = get(dsk, keys, **kwargs)
        393     results_iter = iter(results)
        394     return tuple(a if f is None else f(next(results_iter), *a)


    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/threaded.py in get(dsk, result, cache, num_workers, **kwargs)
         73     results = get_async(pool.apply_async, len(pool._pool), dsk, result,
         74                         cache=cache, get_id=_thread_get_id,
    ---> 75                         pack_exception=pack_exception, **kwargs)
         76 
         77     # Cleanup pools associated to dead threads


    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
        519                         _execute_task(task, data)  # Re-execute locally
        520                     else:
    --> 521                         raise_exception(exc, tb)
        522                 res, worker_id = loads(res_info)
        523                 state['cache'][key] = res


    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/compatibility.py in reraise(exc, tb)
         65         if exc.__traceback__ is not tb:
         66             raise exc.with_traceback(tb)
    ---> 67         raise exc
         68 
         69 else:


    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
        288     try:
        289         task, data = loads(task_info)
    --> 290         result = _execute_task(task, data)
        291         id = get_id()
        292         result = dumps((result, id))


    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/local.py in _execute_task(arg, cache, dsk)
        269         func, args = arg[0], arg[1:]
        270         args2 = [_execute_task(a, cache) for a in args]
    --> 271         return func(*args2)
        272     elif not ishashable(arg):
        273         return arg


    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/dataframe/io/parquet.py in _read_parquet_row_group(fs, fn, index, columns, rg, series, categories, schema, cs, dt, scheme, storage_name_mapping, *args)
        364                               for name in index]
        365     df.columns = [storage_name_mapping.get(col, col)
    --> 366                   for col in columns
        367                   if col != index]
        368 


    ~/andre/yakushima-3env/lib/python3.5/site-packages/pandas/core/generic.py in __setattr__(self, name, value)
       3625         try:
       3626             object.__getattribute__(self, name)
    -> 3627             return object.__setattr__(self, name, value)
       3628         except AttributeError:
       3629             pass


    pandas/_libs/properties.pyx in pandas._libs.properties.AxisProperty.__set__()


    ~/andre/yakushima-3env/lib/python3.5/site-packages/pandas/core/generic.py in _set_axis(self, axis, labels)
        557 
        558     def _set_axis(self, axis, labels):
    --> 559         self._data.set_axis(axis, labels)
        560         self._clear_item_cache()
        561 


    ~/andre/yakushima-3env/lib/python3.5/site-packages/pandas/core/internals.py in set_axis(self, axis, new_labels)
       3072             raise ValueError('Length mismatch: Expected axis has %d elements, '
       3073                              'new values have %d elements' %
    -> 3074                              (old_len, new_len))
       3075 
       3076         self.axes[axis] = new_labels


    ValueError: Length mismatch: Expected axis has 2 elements, new values have 1 elements


I also noticed that I am not able to load one column (even without filtering):


```python
df_partitioned[['fake_categorical1']].compute()
```


    ---------------------------------------------------------------------------

    ValueError                                Traceback (most recent call last)

    <ipython-input-34-a68c843edc71> in <module>()
    ----> 1 df_partitioned[['fake_categorical1']].compute()
    

    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/base.py in compute(self, **kwargs)
        141         dask.base.compute
        142         """
    --> 143         (result,) = compute(self, traverse=False, **kwargs)
        144         return result
        145 


    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/base.py in compute(*args, **kwargs)
        390     postcomputes = [a.__dask_postcompute__() if is_dask_collection(a)
        391                     else (None, a) for a in args]
    --> 392     results = get(dsk, keys, **kwargs)
        393     results_iter = iter(results)
        394     return tuple(a if f is None else f(next(results_iter), *a)


    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/threaded.py in get(dsk, result, cache, num_workers, **kwargs)
         73     results = get_async(pool.apply_async, len(pool._pool), dsk, result,
         74                         cache=cache, get_id=_thread_get_id,
    ---> 75                         pack_exception=pack_exception, **kwargs)
         76 
         77     # Cleanup pools associated to dead threads


    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
        519                         _execute_task(task, data)  # Re-execute locally
        520                     else:
    --> 521                         raise_exception(exc, tb)
        522                 res, worker_id = loads(res_info)
        523                 state['cache'][key] = res


    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/compatibility.py in reraise(exc, tb)
         65         if exc.__traceback__ is not tb:
         66             raise exc.with_traceback(tb)
    ---> 67         raise exc
         68 
         69 else:


    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
        288     try:
        289         task, data = loads(task_info)
    --> 290         result = _execute_task(task, data)
        291         id = get_id()
        292         result = dumps((result, id))


    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/local.py in _execute_task(arg, cache, dsk)
        269         func, args = arg[0], arg[1:]
        270         args2 = [_execute_task(a, cache) for a in args]
    --> 271         return func(*args2)
        272     elif not ishashable(arg):
        273         return arg


    ~/andre/yakushima-3env/lib/python3.5/site-packages/dask/dataframe/io/parquet.py in _read_parquet_row_group(fs, fn, index, columns, rg, series, categories, schema, cs, dt, scheme, storage_name_mapping, *args)
        364                               for name in index]
        365     df.columns = [storage_name_mapping.get(col, col)
    --> 366                   for col in columns
        367                   if col != index]
        368 


    ~/andre/yakushima-3env/lib/python3.5/site-packages/pandas/core/generic.py in __setattr__(self, name, value)
       3625         try:
       3626             object.__getattribute__(self, name)
    -> 3627             return object.__setattr__(self, name, value)
       3628         except AttributeError:
       3629             pass


    pandas/_libs/properties.pyx in pandas._libs.properties.AxisProperty.__set__()


    ~/andre/yakushima-3env/lib/python3.5/site-packages/pandas/core/generic.py in _set_axis(self, axis, labels)
        557 
        558     def _set_axis(self, axis, labels):
    --> 559         self._data.set_axis(axis, labels)
        560         self._clear_item_cache()
        561 


    ~/andre/yakushima-3env/lib/python3.5/site-packages/pandas/core/internals.py in set_axis(self, axis, new_labels)
       3072             raise ValueError('Length mismatch: Expected axis has %d elements, '
       3073                              'new values have %d elements' %
    -> 3074                              (old_len, new_len))
       3075 
       3076         self.axes[axis] = new_labels


    ValueError: Length mismatch: Expected axis has 2 elements, new values have 1 elements




I also noticed that if I choose both columns that were used to do the partitioning it works! But I'm still not able to filter them before computing :(


```python
df_partitioned[['fake_categorical1', 'fake_categorical2']].compute().head()
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>fake_categorical1</th>
      <th>fake_categorical2</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>A</td>
      <td>D</td>
    </tr>
    <tr>
      <th>1</th>
      <td>A</td>
      <td>D</td>
    </tr>
    <tr>
      <th>2</th>
      <td>A</td>
      <td>D</td>
    </tr>
    <tr>
      <th>3</th>
      <td>A</td>
      <td>D</td>
    </tr>
    <tr>
      <th>4</th>
      <td>A</td>
      <td>D</td>
    </tr>
  </tbody>
</table>
</div>


</details>

# Questions

1. Is this a bug?
2. If not, Am I loading and filtering the partitioned parquet correctly? How can I filter on a value of a specific column before mamking `compute()`?


Thank you in advance!



Discussion:
I found a way of how to do it. If I read the partitioned parquet as:

```
df_partitioned = dd.read_parquet('file_partitioned',
                                 filters=[('fake_categorical1', '==', 'A')])
```

I am able to get a DataFrame that only contains `fake_categorical1=='A'`. But still this is not very usable or practical. Because after if I want to run, for example, the following:

```
df_partitioned[(df_partitioned.variable=='signal1')].head()
```

I get the same error.

Is there a way to do it with `.loc` or other way similar to pandas?
Thank you for a well-specified problem. This sounds like a bug, possibly in fastparquet.

Note that using `filters=` is a different mechanism altogether: you do not read the partitions that don't have any rows matching your criteria at all; whereas filtering with `[..]` does consider every row of every partition.

I suspect that the error is coming from the *selection* of one of the columns - there is a special optimization under the hood to only load those columns from the underlying data; but in this case, the special column does not correspond to any data, only directory names, so maybe nothing is being loaded.

One immediate workaround you could try:
```
df_partitioned.map_partitions(lambda df: df[df.variable=='signal1'])
```
which, if it works, will I think will prove what the problem is.
@martindurant Thank you for your reply.

I understand that `filters=` only causes the read of the relevant partitions. But it seems a bit surprising that filtering with `[...]` would *always* cause the read of *all* partitions.

I think it would make sense that when filtering on the column which the parquet file was partitioned on, the **lib** (Dask or Fastparquet, not sure which one should have such abstraction) should be clever enough to know the DataFrame was coming from a partitioned parquet file and to only 'touch' the relevant partitions.

For example:

```Python
fastparquet.write('file_partitioned', small_df, compression='snappy',
                  file_scheme='hive', write_index=False,
                  partition_on=['fake_categorical1','fake_categorical2'])

df_partitioned = dd.read_parquet('file_partitioned')

pandas_df = df_partitioned[(df_partitioned.fake_categorical2 =='A') &
                        ((df_partitioned.variable=='signal1'))].compute()
```

Should only read the partitions where `fake_categorical2 =='A'` and then filter those partitions on `variable=='signal1'`.

Do you agree?


As for your workaround I'll only be able to test it tomorrow. But if  am not mistaken, if it works, it will cause the load of *all* partitions, even if I am filtering on the column that was used to create the partitions, right?



Resolving patch:
diff --git a/dask/dataframe/io/parquet.py b/dask/dataframe/io/parquet.py
--- a/dask/dataframe/io/parquet.py
+++ b/dask/dataframe/io/parquet.py
@@ -339,6 +339,8 @@ def _read_parquet_row_group(fs, fn, index, columns, rg, series, categories,
                             schema, cs, dt, scheme, storage_name_mapping, *args):
     from fastparquet.api import _pre_allocate
     from fastparquet.core import read_row_group_file
+    from collections import OrderedDict
+
     name_storage_mapping = {v: k for k, v in storage_name_mapping.items()}
     if not isinstance(columns, (tuple, list)):
         columns = [columns,]
@@ -350,6 +352,7 @@ def _read_parquet_row_group(fs, fn, index, columns, rg, series, categories,
 
     columns = [name_storage_mapping.get(col, col) for col in columns]
     index = name_storage_mapping.get(index, index)
+    cs = OrderedDict([(k, v) for k, v in cs.items() if k in columns])
 
     df, views = _pre_allocate(rg.num_rows, columns, categories, index, cs, dt)
     read_row_group_file(fn, rg, columns, categories, schema, cs,


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -1025,3 +1025,19 @@ def test_writing_parquet_with_unknown_kwargs(tmpdir, engine):
 
     with pytest.raises(TypeError):
         ddf.to_parquet(fn,  engine=engine, unknown_key='unknown_value')
+
+
+def test_setect_partitioned_column(tmpdir, engine):
+    if engine == 'pyarrow':
+        pytest.xfail()
+    fn = str(tmpdir)
+    size = 20
+    d = {'signal1': np.random.normal(0, 0.3, size=size).cumsum() + 50,
+         'fake_categorical1': np.random.choice(['A', 'B', 'C'], size=size),
+         'fake_categorical2': np.random.choice(['D', 'E', 'F'], size=size)}
+    df = dd.from_pandas(pd.DataFrame(d), 2)
+    df.to_parquet(fn, compression='snappy', write_index=False, engine=engine,
+                  partition_on=['fake_categorical1', 'fake_categorical2'])
+
+    df_partitioned = dd.read_parquet(fn, engine=engine)
+    df_partitioned[df_partitioned.fake_categorical1 == 'A'].compute()