Repository: dask/dask

Issue:
Dask 2.1.0, KeyError: 'Column not found: 0' ? 
I have a data frame with varying types and am getting the subject error when trying to groupby
After review of some posts on meta and am still not certain where to add meta, since both from_pandas(df, meta) and other attempts fail.


```
for idx,grp in ddf.groupby(['f1','f2',ddf.dt64.dt.dayofweek,ddf.dt64.dt.hour,ddf.dt64.dt.minute]):
....

```

```
C:\g\test\lib\dask\dataframe\groupby.py in __getitem__(self, key)
   1523 
   1524         # error is raised from pandas
-> 1525         g._meta = g._meta[key]
   1526         return g
   1527 

C:\g\test\lib\pandas\core\base.py in __getitem__(self, key)
    266         else:
    267             if key not in self.obj:
--> 268                 raise KeyError("Column not found: {key}".format(key=key))
    269             return self._gotitem(key, ndim=1)
    270 

KeyError: 'Column not found: 0'
```



Discussion:
Can you provide a minimal example? http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports
```
import pandas as pd
import dask.dataframe as daskdf
import numpy as np
pdf=pd.DataFrame(np.random.randint(0,100,size=(10,8)),columns=list('abcdefgh'))
for idx,grp in pdf.groupby(list('abcdefgh')[:6]):
    print(idx)
    print(grp,'\n')

print("DASK")
ddf=daskdf.from_pandas(pdf,npartitions=2)
for idx,grp in ddf.groupby(list('abcdefgh')[:6]):
    print(idx)
    print(grp,'\n')

(8, 2, 5, 36, 35, 43)
   a  b  c   d   e   f   g   h
0  8  2  5  36  35  43  41  41 

(20, 39, 48, 45, 0, 47)
    a   b   c   d  e   f   g   h
2  20  39  48  45  0  47  23  44 

(36, 17, 28, 69, 1, 15)
    a   b   c   d  e   f   g   h
7  36  17  28  69  1  15  10  56 

(39, 57, 99, 10, 33, 13)
    a   b   c   d   e   f   g   h
4  39  57  99  10  33  13  85  66 

(41, 92, 47, 0, 63, 42)
    a   b   c  d   e   f   g   h
8  41  92  47  0  63  42  33  28 

(45, 24, 60, 26, 81, 23)
    a   b   c   d   e   f   g   h
5  45  24  60  26  81  23  30  19 

(62, 43, 57, 94, 89, 67)
    a   b   c   d   e   f   g   h
9  62  43  57  94  89  67  85  10 

(70, 53, 82, 39, 5, 83)
    a   b   c   d  e   f   g   h
6  70  53  82  39  5  83  53  95 

(86, 59, 18, 96, 5, 66)
    a   b   c   d  e   f   g   h
3  86  59  18  96  5  66  91  92 

(90, 68, 65, 17, 21, 14)
    a   b   c   d   e   f   g   h
1  90  68  65  17  21  14  33  46 

DASK
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-10-aea283cedd26> in <module>
      9 print("DASK")
     10 ddf=daskdf.from_pandas(pdf,npartitions=2)
---> 11 for idx,grp in ddf.groupby(list('abcdefgh')[:6]):
     12     print(idx)
     13     print(grp,'\n')

C:\g\via\lib\dask\dataframe\groupby.py in __getitem__(self, key)
   1523 
   1524         # error is raised from pandas
-> 1525         g._meta = g._meta[key]
   1526         return g
   1527 

C:\g\via\lib\pandas\core\base.py in __getitem__(self, key)
    266         else:
    267             if key not in self.obj:
--> 268                 raise KeyError("Column not found: {key}".format(key=key))
    269             return self._gotitem(key, ndim=1)
    270 

KeyError: 'Column not found: 0'

```
Ah, thanks.

So dask's DataFrameGroupBy doesn't currently implement __iter__. Would you
be interested in looking into whether it can be implemented?

If it doesn't look feasible, we can implement a `__iter__` that raises a
NotImplementedError instead of this confusing error message.

On Sat, Jul 20, 2019 at 11:39 AM apiszcz <notifications@github.com> wrote:

> import pandas as pd
> import dask.dataframe as daskdf
> import numpy as np
> pdf=pd.DataFrame(np.random.randint(0,100,size=(10,8)),columns=list('abcdefgh'))
> for idx,grp in pdf.groupby(list('abcdefgh')[:6]):
>     print(idx)
>     print(grp,'\n')
>
> print("DASK")
> ddf=daskdf.from_pandas(pdf,npartitions=2)
> for idx,grp in ddf.groupby(list('abcdefgh')[:6]):
>     print(idx)
>     print(grp,'\n')
>
> (8, 2, 5, 36, 35, 43)
>    a  b  c   d   e   f   g   h
> 0  8  2  5  36  35  43  41  41
>
> (20, 39, 48, 45, 0, 47)
>     a   b   c   d  e   f   g   h
> 2  20  39  48  45  0  47  23  44
>
> (36, 17, 28, 69, 1, 15)
>     a   b   c   d  e   f   g   h
> 7  36  17  28  69  1  15  10  56
>
> (39, 57, 99, 10, 33, 13)
>     a   b   c   d   e   f   g   h
> 4  39  57  99  10  33  13  85  66
>
> (41, 92, 47, 0, 63, 42)
>     a   b   c  d   e   f   g   h
> 8  41  92  47  0  63  42  33  28
>
> (45, 24, 60, 26, 81, 23)
>     a   b   c   d   e   f   g   h
> 5  45  24  60  26  81  23  30  19
>
> (62, 43, 57, 94, 89, 67)
>     a   b   c   d   e   f   g   h
> 9  62  43  57  94  89  67  85  10
>
> (70, 53, 82, 39, 5, 83)
>     a   b   c   d  e   f   g   h
> 6  70  53  82  39  5  83  53  95
>
> (86, 59, 18, 96, 5, 66)
>     a   b   c   d  e   f   g   h
> 3  86  59  18  96  5  66  91  92
>
> (90, 68, 65, 17, 21, 14)
>     a   b   c   d   e   f   g   h
> 1  90  68  65  17  21  14  33  46
>
> DASK
> ---------------------------------------------------------------------------
> KeyError                                  Traceback (most recent call last)
> <ipython-input-10-aea283cedd26> in <module>
>       9 print("DASK")
>      10 ddf=daskdf.from_pandas(pdf,npartitions=2)
> ---> 11 for idx,grp in ddf.groupby(list('abcdefgh')[:6]):
>      12     print(idx)
>      13     print(grp,'\n')
>
> C:\g\via\lib\dask\dataframe\groupby.py in __getitem__(self, key)
>    1523
>    1524         # error is raised from pandas
> -> 1525         g._meta = g._meta[key]
>    1526         return g
>    1527
>
> C:\g\via\lib\pandas\core\base.py in __getitem__(self, key)
>     266         else:
>     267             if key not in self.obj:
> --> 268                 raise KeyError("Column not found: {key}".format(key=key))
>     269             return self._gotitem(key, ndim=1)
>     270
>
> KeyError: 'Column not found: 0'
>
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/5124?email_source=notifications&email_token=AAKAOIXT32QEIUHGD6OR6ULQAM5V3A5CNFSM4IFOHBLKYY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOD2NRY5Y#issuecomment-513481847>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AAKAOITDFMCTSLKCIN34XLTQAM5V3ANCNFSM4IFOHBLA>
> .
>

Yes, if feasible, I'm unsure how much effort this requires and it would
seem natural to emulate the pandas API, so I'm speculating if it was
straightforward, it would already have an iterator?

On Mon, Jul 22, 2019 at 12:13 PM Tom Augspurger <notifications@github.com>
wrote:

> Ah, thanks.
>
> So dask's DataFrameGroupBy doesn't currently implement __iter__. Would you
> be interested in looking into whether it can be implemented?
>
> If it doesn't look feasible, we can implement a `__iter__` that raises a
> NotImplementedError instead of this confusing error message.
>
> On Sat, Jul 20, 2019 at 11:39 AM apiszcz <notifications@github.com> wrote:
>
> > import pandas as pd
> > import dask.dataframe as daskdf
> > import numpy as np
> >
> pdf=pd.DataFrame(np.random.randint(0,100,size=(10,8)),columns=list('abcdefgh'))
> > for idx,grp in pdf.groupby(list('abcdefgh')[:6]):
> > print(idx)
> > print(grp,'\n')
> >
> > print("DASK")
> > ddf=daskdf.from_pandas(pdf,npartitions=2)
> > for idx,grp in ddf.groupby(list('abcdefgh')[:6]):
> > print(idx)
> > print(grp,'\n')
> >
> > (8, 2, 5, 36, 35, 43)
> > a b c d e f g h
> > 0 8 2 5 36 35 43 41 41
> >
> > (20, 39, 48, 45, 0, 47)
> > a b c d e f g h
> > 2 20 39 48 45 0 47 23 44
> >
> > (36, 17, 28, 69, 1, 15)
> > a b c d e f g h
> > 7 36 17 28 69 1 15 10 56
> >
> > (39, 57, 99, 10, 33, 13)
> > a b c d e f g h
> > 4 39 57 99 10 33 13 85 66
> >
> > (41, 92, 47, 0, 63, 42)
> > a b c d e f g h
> > 8 41 92 47 0 63 42 33 28
> >
> > (45, 24, 60, 26, 81, 23)
> > a b c d e f g h
> > 5 45 24 60 26 81 23 30 19
> >
> > (62, 43, 57, 94, 89, 67)
> > a b c d e f g h
> > 9 62 43 57 94 89 67 85 10
> >
> > (70, 53, 82, 39, 5, 83)
> > a b c d e f g h
> > 6 70 53 82 39 5 83 53 95
> >
> > (86, 59, 18, 96, 5, 66)
> > a b c d e f g h
> > 3 86 59 18 96 5 66 91 92
> >
> > (90, 68, 65, 17, 21, 14)
> > a b c d e f g h
> > 1 90 68 65 17 21 14 33 46
> >
> > DASK
> >
> ---------------------------------------------------------------------------
> > KeyError Traceback (most recent call last)
> > <ipython-input-10-aea283cedd26> in <module>
> > 9 print("DASK")
> > 10 ddf=daskdf.from_pandas(pdf,npartitions=2)
> > ---> 11 for idx,grp in ddf.groupby(list('abcdefgh')[:6]):
> > 12 print(idx)
> > 13 print(grp,'\n')
> >
> > C:\g\via\lib\dask\dataframe\groupby.py in __getitem__(self, key)
> > 1523
> > 1524 # error is raised from pandas
> > -> 1525 g._meta = g._meta[key]
> > 1526 return g
> > 1527
> >
> > C:\g\via\lib\pandas\core\base.py in __getitem__(self, key)
> > 266 else:
> > 267 if key not in self.obj:
> > --> 268 raise KeyError("Column not found: {key}".format(key=key))
> > 269 return self._gotitem(key, ndim=1)
> > 270
> >
> > KeyError: 'Column not found: 0'
> >
> >
> > —
> > You are receiving this because you commented.
> > Reply to this email directly, view it on GitHub
> > <
> https://github.com/dask/dask/issues/5124?email_source=notifications&email_token=AAKAOIXT32QEIUHGD6OR6ULQAM5V3A5CNFSM4IFOHBLKYY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOD2NRY5Y#issuecomment-513481847
> >,
> > or mute the thread
> > <
> https://github.com/notifications/unsubscribe-auth/AAKAOITDFMCTSLKCIN34XLTQAM5V3ANCNFSM4IFOHBLA
> >
> > .
> >
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/5124?email_source=notifications&email_token=AAK5KTKMETWX7MQARM4HWK3QAXMCFA5CNFSM4IFOHBLKYY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOD2QLYYQ#issuecomment-513850466>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AAK5KTKCPTM5R24KZ3IDJPLQAXMCFANCNFSM4IFOHBLA>
> .
>

On Mon, Jul 22, 2019 at 1:53 PM apiszcz <notifications@github.com> wrote:

> Yes, if feasible, I'm unsure how much effort this requires and it would
> seem natural to emulate the pandas API, so I'm speculating if it was
> straightforward, it would already have an iterator?
>

Not necessarily. We tend to implementing things based on user feedback. But
sometimes a part of the Pandas API
slips through.


> On Mon, Jul 22, 2019 at 12:13 PM Tom Augspurger <notifications@github.com>
> wrote:
>
> > Ah, thanks.
> >
> > So dask's DataFrameGroupBy doesn't currently implement __iter__. Would
> you
> > be interested in looking into whether it can be implemented?
> >
> > If it doesn't look feasible, we can implement a `__iter__` that raises a
> > NotImplementedError instead of this confusing error message.
> >
> > On Sat, Jul 20, 2019 at 11:39 AM apiszcz <notifications@github.com>
> wrote:
> >
> > > import pandas as pd
> > > import dask.dataframe as daskdf
> > > import numpy as np
> > >
> >
> pdf=pd.DataFrame(np.random.randint(0,100,size=(10,8)),columns=list('abcdefgh'))
> > > for idx,grp in pdf.groupby(list('abcdefgh')[:6]):
> > > print(idx)
> > > print(grp,'\n')
> > >
> > > print("DASK")
> > > ddf=daskdf.from_pandas(pdf,npartitions=2)
> > > for idx,grp in ddf.groupby(list('abcdefgh')[:6]):
> > > print(idx)
> > > print(grp,'\n')
> > >
> > > (8, 2, 5, 36, 35, 43)
> > > a b c d e f g h
> > > 0 8 2 5 36 35 43 41 41
> > >
> > > (20, 39, 48, 45, 0, 47)
> > > a b c d e f g h
> > > 2 20 39 48 45 0 47 23 44
> > >
> > > (36, 17, 28, 69, 1, 15)
> > > a b c d e f g h
> > > 7 36 17 28 69 1 15 10 56
> > >
> > > (39, 57, 99, 10, 33, 13)
> > > a b c d e f g h
> > > 4 39 57 99 10 33 13 85 66
> > >
> > > (41, 92, 47, 0, 63, 42)
> > > a b c d e f g h
> > > 8 41 92 47 0 63 42 33 28
> > >
> > > (45, 24, 60, 26, 81, 23)
> > > a b c d e f g h
> > > 5 45 24 60 26 81 23 30 19
> > >
> > > (62, 43, 57, 94, 89, 67)
> > > a b c d e f g h
> > > 9 62 43 57 94 89 67 85 10
> > >
> > > (70, 53, 82, 39, 5, 83)
> > > a b c d e f g h
> > > 6 70 53 82 39 5 83 53 95
> > >
> > > (86, 59, 18, 96, 5, 66)
> > > a b c d e f g h
> > > 3 86 59 18 96 5 66 91 92
> > >
> > > (90, 68, 65, 17, 21, 14)
> > > a b c d e f g h
> > > 1 90 68 65 17 21 14 33 46
> > >
> > > DASK
> > >
> >
> ---------------------------------------------------------------------------
> > > KeyError Traceback (most recent call last)
> > > <ipython-input-10-aea283cedd26> in <module>
> > > 9 print("DASK")
> > > 10 ddf=daskdf.from_pandas(pdf,npartitions=2)
> > > ---> 11 for idx,grp in ddf.groupby(list('abcdefgh')[:6]):
> > > 12 print(idx)
> > > 13 print(grp,'\n')
> > >
> > > C:\g\via\lib\dask\dataframe\groupby.py in __getitem__(self, key)
> > > 1523
> > > 1524 # error is raised from pandas
> > > -> 1525 g._meta = g._meta[key]
> > > 1526 return g
> > > 1527
> > >
> > > C:\g\via\lib\pandas\core\base.py in __getitem__(self, key)
> > > 266 else:
> > > 267 if key not in self.obj:
> > > --> 268 raise KeyError("Column not found: {key}".format(key=key))
> > > 269 return self._gotitem(key, ndim=1)
> > > 270
> > >
> > > KeyError: 'Column not found: 0'
> > >
> > >
> > > —
> > > You are receiving this because you commented.
> > > Reply to this email directly, view it on GitHub
> > > <
> >
> https://github.com/dask/dask/issues/5124?email_source=notifications&email_token=AAKAOIXT32QEIUHGD6OR6ULQAM5V3A5CNFSM4IFOHBLKYY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOD2NRY5Y#issuecomment-513481847
> > >,
> > > or mute the thread
> > > <
> >
> https://github.com/notifications/unsubscribe-auth/AAKAOITDFMCTSLKCIN34XLTQAM5V3ANCNFSM4IFOHBLA
> > >
> > > .
> > >
> >
> > —
> > You are receiving this because you authored the thread.
> > Reply to this email directly, view it on GitHub
> > <
> https://github.com/dask/dask/issues/5124?email_source=notifications&email_token=AAK5KTKMETWX7MQARM4HWK3QAXMCFA5CNFSM4IFOHBLKYY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOD2QLYYQ#issuecomment-513850466
> >,
> > or mute the thread
> > <
> https://github.com/notifications/unsubscribe-auth/AAK5KTKCPTM5R24KZ3IDJPLQAXMCFANCNFSM4IFOHBLA
> >
> > .
> >
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/5124?email_source=notifications&email_token=AAKAOIT6WUWXJFORC37BZFLQAX63RA5CNFSM4IFOHBLKYY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOD2Q2EXA#issuecomment-513909340>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AAKAOIQ2S5Y3DYLGEPL2RSLQAX63RANCNFSM4IFOHBLA>
> .
>

Dask Groupby does not support iterator.

On Sun, Jul 28, 2019 at 7:40 PM Craig Evans <notifications@github.com>
wrote:

> I have a similar issue, but not sure it is the same. I am attempting to do
> a simple groupby on a dask dataframe but receive a KeyError: 'Column not
> found: 0'
>
> I have tested the following code with dask 1.2.2 and 2.1.0 and have the
> same result
> *the code used to test both on a pandas dataframe and a dask dataframe:*
>
> import pandas as pdfrom dask import dataframe as ddfrom dask import multiprocessingfrom dask.distributed import Client
>
> client = Client(processes=False)
>
> data = {
>     'col1': [1, 1, 1, 2, 2, 2, 3, 3, 3],
>     'col2': ['apple','bananna','orange','apple','bananna','orange','apple','bananna','orange'],
>     'col3': [34, 12, 1, 36, 22, 6, 22, 16, 4]
>     }
> pdf = pd.DataFrame(data=data)print('*************  Pandas DataFrame')print(pdf.head(5))
> print('')print('Performing groupby on Pandas DataFrame')
> pgroup = pdf.groupby(by='col2')for name, group in pgroup:
>     print('')
>     print(f'Group: {name}')
>     print(group.head(5))
> print(' ')print(' ')
>
> ddf = dd.from_pandas(data=pdf, npartitions=1)print('*************  Dask DataFrame')print(ddf.head(5))
> print('')print('Performing groupby on Dask DataFrame')
> dgroup = ddf.groupby(by='col2')for name, group in dgroup:
>     print('')
>     print(f'Group: {name}')
>     print(group.head(5))
>
> *I receive the following output and error*
>
> *************  Pandas DataFrame
>    col1     col2  col3
> 0     1    apple    34
> 1     1  bananna    12
> 2     1   orange     1
> 3     2    apple    36
> 4     2  bananna    22
>
> Performing groupby on Pandas DataFrame
>
> Group: apple
>    col1   col2  col3
> 0     1  apple    34
> 3     2  apple    36
> 6     3  apple    22
>
> Group: bananna
>    col1     col2  col3
> 1     1  bananna    12
> 4     2  bananna    22
> 7     3  bananna    16
>
> Group: orange
>    col1    col2  col3
> 2     1  orange     1
> 5     2  orange     6
> 8     3  orange     4
>
>
> *************  Dask DataFrame
>    col1     col2  col3
> 0     1    apple    34
> 1     1  bananna    12
> 2     1   orange     1
> 3     2    apple    36
> 4     2  bananna    22
>
> Performing groupby on Dask DataFrame
> Traceback (most recent call last):
>   File "C:\Users\Craig\source\repos\cevans3098\MarketData_preProcessor\module1.py", line 37, in <module>
>     for name, group in dgroup:
>   File "F:\anaconda3\lib\site-packages\dask\dataframe\groupby.py", line 1525, in __getitem__
>     g._meta = g._meta[key]
>   File "F:\anaconda3\lib\site-packages\pandas\core\base.py", line 275, in __getitem__
>     raise KeyError("Column not found: {key}".format(key=key))
> KeyError: 'Column not found: 0'
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/5124?email_source=notifications&email_token=AAK5KTM56W7DA4LT6KUQFVDQBYVANA5CNFSM4IFOHBLKYY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOD27I46A#issuecomment-515804792>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AAK5KTMLTYXK6B5UQ32Z4KLQBYVANANCNFSM4IFOHBLA>
> .
>

Just do be clear, rather than "does not support" I'd say "doesn't currently
implement". It's in scope and should be doable.

On Mon, Jul 29, 2019 at 6:01 AM apiszcz <notifications@github.com> wrote:

> Dask Groupby does not support iterator.
>
> On Sun, Jul 28, 2019 at 7:40 PM Craig Evans <notifications@github.com>
> wrote:
>
> > I have a similar issue, but not sure it is the same. I am attempting to
> do
> > a simple groupby on a dask dataframe but receive a KeyError: 'Column not
> > found: 0'
> >
> > I have tested the following code with dask 1.2.2 and 2.1.0 and have the
> > same result
> > *the code used to test both on a pandas dataframe and a dask dataframe:*
> >
> > import pandas as pdfrom dask import dataframe as ddfrom dask import
> multiprocessingfrom dask.distributed import Client
> >
> > client = Client(processes=False)
> >
> > data = {
> > 'col1': [1, 1, 1, 2, 2, 2, 3, 3, 3],
> > 'col2':
> ['apple','bananna','orange','apple','bananna','orange','apple','bananna','orange'],
> > 'col3': [34, 12, 1, 36, 22, 6, 22, 16, 4]
> > }
> > pdf = pd.DataFrame(data=data)print('************* Pandas
> DataFrame')print(pdf.head(5))
> > print('')print('Performing groupby on Pandas DataFrame')
> > pgroup = pdf.groupby(by='col2')for name, group in pgroup:
> > print('')
> > print(f'Group: {name}')
> > print(group.head(5))
> > print(' ')print(' ')
> >
> > ddf = dd.from_pandas(data=pdf, npartitions=1)print('************* Dask
> DataFrame')print(ddf.head(5))
> > print('')print('Performing groupby on Dask DataFrame')
> > dgroup = ddf.groupby(by='col2')for name, group in dgroup:
> > print('')
> > print(f'Group: {name}')
> > print(group.head(5))
> >
> > *I receive the following output and error*
> >
> > ************* Pandas DataFrame
> > col1 col2 col3
> > 0 1 apple 34
> > 1 1 bananna 12
> > 2 1 orange 1
> > 3 2 apple 36
> > 4 2 bananna 22
> >
> > Performing groupby on Pandas DataFrame
> >
> > Group: apple
> > col1 col2 col3
> > 0 1 apple 34
> > 3 2 apple 36
> > 6 3 apple 22
> >
> > Group: bananna
> > col1 col2 col3
> > 1 1 bananna 12
> > 4 2 bananna 22
> > 7 3 bananna 16
> >
> > Group: orange
> > col1 col2 col3
> > 2 1 orange 1
> > 5 2 orange 6
> > 8 3 orange 4
> >
> >
> > ************* Dask DataFrame
> > col1 col2 col3
> > 0 1 apple 34
> > 1 1 bananna 12
> > 2 1 orange 1
> > 3 2 apple 36
> > 4 2 bananna 22
> >
> > Performing groupby on Dask DataFrame
> > Traceback (most recent call last):
> > File
> "C:\Users\Craig\source\repos\cevans3098\MarketData_preProcessor\module1.py",
> line 37, in <module>
> > for name, group in dgroup:
> > File "F:\anaconda3\lib\site-packages\dask\dataframe\groupby.py", line
> 1525, in __getitem__
> > g._meta = g._meta[key]
> > File "F:\anaconda3\lib\site-packages\pandas\core\base.py", line 275, in
> __getitem__
> > raise KeyError("Column not found: {key}".format(key=key))
> > KeyError: 'Column not found: 0'
> >
> > —
> > You are receiving this because you authored the thread.
> > Reply to this email directly, view it on GitHub
> > <
> https://github.com/dask/dask/issues/5124?email_source=notifications&email_token=AAK5KTM56W7DA4LT6KUQFVDQBYVANA5CNFSM4IFOHBLKYY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOD27I46A#issuecomment-515804792
> >,
> > or mute the thread
> > <
> https://github.com/notifications/unsubscribe-auth/AAK5KTMLTYXK6B5UQ32Z4KLQBYVANANCNFSM4IFOHBLA
> >
> > .
> >
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/5124?email_source=notifications&email_token=AAKAOIUNOQCFGGJD4SY4RVLQB3EXHA5CNFSM4IFOHBLKYY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOD3ALO2Q#issuecomment-515946346>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AAKAOIV3EPBVWFMEWIIK2OLQB3EXHANCNFSM4IFOHBLA>
> .
>

How can I then iterate through all groups as separate dataframes?
As pointed out in https://github.com/dask/dask/issues/5124#issuecomment-513850466, Dask's `DataFrameGroupBy` doesn't currently implement `__iter__`. But contributions are always welcome! 
I tried to use `get_group` method, but then `compute` produces another `KeyError`:
```python
import pandas as pd
import dask.dataframe
df = pd.DataFrame(
    [[1, "foo"], [2, "bar"], [3, "foo"], [4, "bar"], [5, "foo"]],
    columns=["id", "type"],
)
x = dask.dataframe.from_pandas(df, npartitions=1)

y = x.groupby("type")
z = y.get_group("foo")
z.compute()  # KeyError: 't'
```
The key mentioned as missing by the KeyError is always the first character of the grouping column.
However, this works OK:
```python
y = x.groupby(["type"])  # column name inside list
z = y.get_group("foo")
z.compute()  # no error
```
Thanks for the nice example @adampl, I'm able to reproduce the same `KeyError`. I believe this is a duplicate of https://github.com/dask/dask/issues/5080 and a fix along the lines of https://github.com/dask/dask/issues/5080#issuecomment-509391613 would be very welcome if you have an interest in submitting a PR (no obligation though)
Is there a solution to this yet ? I want to iterate over the dataframes and I can't use the get_group() method as I don't have the group names(and there are a lot of them) 
This isn't implemented yet.

Doing so might be somewhat difficult / surprising for users. In order to support something like

```python
df.groupby("type").__iter__
```

We need to fully load the data to discover all the unique values in `type`. Users might not be expecting something like `__iter__` to be that expensive.

But that said, if people really do want this then we'd accept a contribution as long as the docs clearly spell out the performance costs.

>  I can't use the get_group() method as I don't have the group names(and there are a lot of them)

Does something like `df['type'].unique().compute()` give you what you need?
Yes I was saving unique values to a list and iterating over that for the time being but I was hoping to avoid that and just iterate over groups like we do with pandas. 
Doesn't getting one group using get_group() require dask to go through all the data? It seems to me that getting groups again and again from dask would be more costly as dask would have to go through the data each time I call a group by. Does this happen or am I missing something? 
I'm not sure how that's implemented, but I suspect you're correct.

On Fri, Jun 19, 2020 at 12:28 AM PalakHarwani <notifications@github.com>
wrote:

> Yes I was saving unique values to a list and iterating over that for the
> time being but I was hoping to avoid that and just iterate over groups like
> we do with pandas.
> Doesn't getting one group using get_group() require dask to go through all
> the data? It seems to me that getting groups again and again from dask
> would be more costly as dask would have to go through the data each time I
> call a group by. Does this happen or am I missing something?
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/5124#issuecomment-646442816>, or
> unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AAKAOISPF54KG7XCLIDRZW3RXLZQXANCNFSM4IFOHBLA>
> .
>

Iterating using unique values and calling `get_group()` is indeed much slower than `group_by["col_name"].apply(function)` (10 times slower in my case)

Resolving patch:
diff --git a/dask/dataframe/groupby.py b/dask/dataframe/groupby.py
--- a/dask/dataframe/groupby.py
+++ b/dask/dataframe/groupby.py
@@ -37,8 +37,8 @@
 #
 # Dask groupby supports reductions, i.e., mean, sum and alike, and apply. The
 # former do not shuffle the data and are efficiently implemented as tree
-# reductions. The latter is implemented by shuffling the underlying partiitons
-# such that all items of a group can be found in the same parititon.
+# reductions. The latter is implemented by shuffling the underlying partitions
+# such that all items of a group can be found in the same partition.
 #
 # The argument to ``.groupby`` (``by``), can be a ``str``, ``dd.DataFrame``,
 # ``dd.Series``, or a list thereof. In operations on the grouped object, the
@@ -1124,6 +1124,14 @@ def _groupby_kwargs(self):
             **self.observed,
         }
 
+    def __iter__(self):
+        raise NotImplementedError(
+            "Iteration of DataFrameGroupBy objects requires computing the groups which "
+            "may be slow. You probably want to use 'apply' to execute a function for "
+            "all the columns. To access individual groups, use 'get_group'. To list "
+            "all the group names, use 'df[<group column>].unique().compute()'."
+        )
+
     @property
     def _meta_nonempty(self):
         """


Test patch:
diff --git a/dask/dataframe/tests/test_groupby.py b/dask/dataframe/tests/test_groupby.py
--- a/dask/dataframe/tests/test_groupby.py
+++ b/dask/dataframe/tests/test_groupby.py
@@ -2608,3 +2608,18 @@ def test_groupby_multi_index_with_row_operations(operation):
     ddf = dd.from_pandas(df, npartitions=3)
     actual = caller(ddf.groupby(["A", ddf["A"].eq("a1")])["B"])
     assert_eq(expected, actual)
+
+
+def test_groupby_iter_fails():
+    df = pd.DataFrame(
+        data=[
+            ["a0", "b1"],
+            ["a1", "b1"],
+            ["a3", "b3"],
+            ["a5", "b5"],
+        ],
+        columns=["A", "B"],
+    )
+    ddf = dd.from_pandas(df, npartitions=1)
+    with pytest.raises(NotImplementedError, match="computing the groups"):
+        list(ddf.groupby("A"))