Repository: dask/dask

Issue:
dataframe merge returns incorrect result
Dask's `dd.merge` operation returns incorrect results under certain unkown condition. Seems to be related to the size of the dataframe. 

I have created a repo for this issue. 
https://gitlab.com/LouisTian/dask_merge_issue



```python
import dask.dataframe as dd
import pandas as pd
```


```python
dask_df = dd.read_csv('./fastText_extract.csv', encoding='utf-8', usecols=list(range(1, 302)))
pd_df = pd.read_csv('./fastText_extract.csv', encoding='utf-8', usecols=list(range(1, 302)))
features = pd.read_csv('./features.csv', encoding='utf-8', usecols=[1])
```


```python
pd_df[pd_df['word'] == 'leonid']
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>word</th>
      <th>0</th>
      <th>1</th>
      <th>2</th>
      <th>3</th>
      <th>4</th>
      <th>5</th>
      <th>6</th>
      <th>7</th>
      <th>8</th>
      <th>...</th>
      <th>290</th>
      <th>291</th>
      <th>292</th>
      <th>293</th>
      <th>294</th>
      <th>295</th>
      <th>296</th>
      <th>297</th>
      <th>298</th>
      <th>299</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>15810</th>
      <td>leonid</td>
      <td>-0.11284</td>
      <td>0.27493</td>
      <td>-0.11428</td>
      <td>0.27465</td>
      <td>-0.1484</td>
      <td>-0.16222</td>
      <td>0.54122</td>
      <td>-0.35387</td>
      <td>-0.44765</td>
      <td>...</td>
      <td>-0.35554</td>
      <td>-0.21266</td>
      <td>0.048219</td>
      <td>-0.023976</td>
      <td>0.23689</td>
      <td>-0.11721</td>
      <td>0.012078</td>
      <td>-0.15143</td>
      <td>-0.19005</td>
      <td>0.043254</td>
    </tr>
  </tbody>
</table>
<p>1 rows × 301 columns</p>
</div>




```python
features[features['word'] == 'leonid']
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>word</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>614</th>
      <td>leonid</td>
    </tr>
  </tbody>
</table>
</div>



# Different merge result using pandas and dask


```python
result = pd_df.merge(features, on='word', how='right')
result[result['word'] == 'leonid']
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>word</th>
      <th>0</th>
      <th>1</th>
      <th>2</th>
      <th>3</th>
      <th>4</th>
      <th>5</th>
      <th>6</th>
      <th>7</th>
      <th>8</th>
      <th>...</th>
      <th>290</th>
      <th>291</th>
      <th>292</th>
      <th>293</th>
      <th>294</th>
      <th>295</th>
      <th>296</th>
      <th>297</th>
      <th>298</th>
      <th>299</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>15810</th>
      <td>leonid</td>
      <td>-0.11284</td>
      <td>0.27493</td>
      <td>-0.11428</td>
      <td>0.27465</td>
      <td>-0.1484</td>
      <td>-0.16222</td>
      <td>0.54122</td>
      <td>-0.35387</td>
      <td>-0.44765</td>
      <td>...</td>
      <td>-0.35554</td>
      <td>-0.21266</td>
      <td>0.048219</td>
      <td>-0.023976</td>
      <td>0.23689</td>
      <td>-0.11721</td>
      <td>0.012078</td>
      <td>-0.15143</td>
      <td>-0.19005</td>
      <td>0.043254</td>
    </tr>
  </tbody>
</table>
<p>1 rows × 301 columns</p>
</div>




```python
result = dask_df.merge(features, on='word', how='right').compute()
result[result['word'] == 'leonid']
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>word</th>
      <th>0</th>
      <th>1</th>
      <th>2</th>
      <th>3</th>
      <th>4</th>
      <th>5</th>
      <th>6</th>
      <th>7</th>
      <th>8</th>
      <th>...</th>
      <th>290</th>
      <th>291</th>
      <th>292</th>
      <th>293</th>
      <th>294</th>
      <th>295</th>
      <th>296</th>
      <th>297</th>
      <th>298</th>
      <th>299</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>9748</th>
      <td>leonid</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>...</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
    </tr>
  </tbody>
</table>
<p>1 rows × 301 columns</p>
</div>



# Use smaller subset of data, same result 


```python
dask_df = dd.read_csv('./fastText_sample.csv', encoding='utf-8', usecols=list(range(1, 302)))
pd_df = pd.read_csv('./fastText_sample.csv', encoding='utf-8', usecols=list(range(1, 302)))
features = pd.read_csv('./features.csv', encoding='utf-8', usecols=[1])
```


```python
result = pd_df.merge(features, on='word', how='right')
result[result['word'] == 'leonid']
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>word</th>
      <th>0</th>
      <th>1</th>
      <th>2</th>
      <th>3</th>
      <th>4</th>
      <th>5</th>
      <th>6</th>
      <th>7</th>
      <th>8</th>
      <th>...</th>
      <th>290</th>
      <th>291</th>
      <th>292</th>
      <th>293</th>
      <th>294</th>
      <th>295</th>
      <th>296</th>
      <th>297</th>
      <th>298</th>
      <th>299</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>810</th>
      <td>leonid</td>
      <td>-0.11284</td>
      <td>0.27493</td>
      <td>-0.11428</td>
      <td>0.27465</td>
      <td>-0.1484</td>
      <td>-0.16222</td>
      <td>0.54122</td>
      <td>-0.35387</td>
      <td>-0.44765</td>
      <td>...</td>
      <td>-0.35554</td>
      <td>-0.21266</td>
      <td>0.048219</td>
      <td>-0.023976</td>
      <td>0.23689</td>
      <td>-0.11721</td>
      <td>0.012078</td>
      <td>-0.15143</td>
      <td>-0.19005</td>
      <td>0.043254</td>
    </tr>
  </tbody>
</table>
<p>1 rows × 301 columns</p>
</div>




```python
result = dask_df.merge(features, on='word', how='right').compute()
result[result['word'] == 'leonid']
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>word</th>
      <th>0</th>
      <th>1</th>
      <th>2</th>
      <th>3</th>
      <th>4</th>
      <th>5</th>
      <th>6</th>
      <th>7</th>
      <th>8</th>
      <th>...</th>
      <th>290</th>
      <th>291</th>
      <th>292</th>
      <th>293</th>
      <th>294</th>
      <th>295</th>
      <th>296</th>
      <th>297</th>
      <th>298</th>
      <th>299</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>810</th>
      <td>leonid</td>
      <td>-0.11284</td>
      <td>0.27493</td>
      <td>-0.11428</td>
      <td>0.27465</td>
      <td>-0.1484</td>
      <td>-0.16222</td>
      <td>0.54122</td>
      <td>-0.35387</td>
      <td>-0.44765</td>
      <td>...</td>
      <td>-0.35554</td>
      <td>-0.21266</td>
      <td>0.048219</td>
      <td>-0.023976</td>
      <td>0.23689</td>
      <td>-0.11721</td>
      <td>0.012078</td>
      <td>-0.15143</td>
      <td>-0.19005</td>
      <td>0.043254</td>
    </tr>
  </tbody>
</table>
<p>1 rows × 301 columns</p>
</div>




```python

```





Discussion:
What version of dask are you using? I looked at that repo (and this one: https://github.com/Louis-Tian/dask_merge_issue) and didn't see the data. If you can provide a [reproducible example](https://stackoverflow.com/help/mcve) and versions used (dask, pandas, partd, and numpy) that would help in debugging.
@jcrist my apology. didn't make the repo public. Data are too big to be pushed to github, so please checkout the [gitlab](https://gitlab.com/LouisTian/dask_merge_issue) repo instead.

I am running 
- python 3.6 
- dask 0.14.1
- pandas 0.19.2
- partd 0.3.7
- numpy 1.12.1

The [requirements.txt](https://gitlab.com/LouisTian/dask_merge_issue/blob/master/requirements.txt) in the repo has the complete environment. 
Run the same code on a Windows machine with python 2.7 and dask started throwing `UnicodeEncodeError` while pandas still runs okay. 

----

```python
import dask.dataframe as dd
import pandas as pd
```


```python
dask_df = dd.read_csv('./fastText_extract.csv', encoding='utf-7', usecols=list(range(1, 302)))
pd_df = pd.read_csv('./fastText_extract.csv', encoding='utf-8', usecols=list(range(1, 302)))
features = pd.read_csv('./features.csv', encoding='utf-8', usecols=[1])
```


```python
pd_df[pd_df['word'] == 'leonid']
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>word</th>
      <th>0</th>
      <th>1</th>
      <th>2</th>
      <th>3</th>
      <th>4</th>
      <th>5</th>
      <th>6</th>
      <th>7</th>
      <th>8</th>
      <th>...</th>
      <th>290</th>
      <th>291</th>
      <th>292</th>
      <th>293</th>
      <th>294</th>
      <th>295</th>
      <th>296</th>
      <th>297</th>
      <th>298</th>
      <th>299</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>15810</th>
      <td>leonid</td>
      <td>-0.11284</td>
      <td>0.27493</td>
      <td>-0.11428</td>
      <td>0.27465</td>
      <td>-0.1484</td>
      <td>-0.16222</td>
      <td>0.54122</td>
      <td>-0.35387</td>
      <td>-0.44765</td>
      <td>...</td>
      <td>-0.35554</td>
      <td>-0.21266</td>
      <td>0.048219</td>
      <td>-0.023976</td>
      <td>0.23689</td>
      <td>-0.11721</td>
      <td>0.012078</td>
      <td>-0.15143</td>
      <td>-0.19005</td>
      <td>0.043254</td>
    </tr>
  </tbody>
</table>
<p>1 rows × 301 columns</p>
</div>




```python
features[features['word'] == 'leonid']
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>word</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>614</th>
      <td>leonid</td>
    </tr>
  </tbody>
</table>
</div>



# Different merge result using pandas and dask


```python
result = pd_df.merge(features, on='word', how='right')
result[result[u'word'] == u'leonid']
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>word</th>
      <th>0</th>
      <th>1</th>
      <th>2</th>
      <th>3</th>
      <th>4</th>
      <th>5</th>
      <th>6</th>
      <th>7</th>
      <th>8</th>
      <th>...</th>
      <th>290</th>
      <th>291</th>
      <th>292</th>
      <th>293</th>
      <th>294</th>
      <th>295</th>
      <th>296</th>
      <th>297</th>
      <th>298</th>
      <th>299</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>15810</th>
      <td>leonid</td>
      <td>-0.11284</td>
      <td>0.27493</td>
      <td>-0.11428</td>
      <td>0.27465</td>
      <td>-0.1484</td>
      <td>-0.16222</td>
      <td>0.54122</td>
      <td>-0.35387</td>
      <td>-0.44765</td>
      <td>...</td>
      <td>-0.35554</td>
      <td>-0.21266</td>
      <td>0.048219</td>
      <td>-0.023976</td>
      <td>0.23689</td>
      <td>-0.11721</td>
      <td>0.012078</td>
      <td>-0.15143</td>
      <td>-0.19005</td>
      <td>0.043254</td>
    </tr>
  </tbody>
</table>
<p>1 rows × 301 columns</p>
</div>




```python
result = dask_df.merge(features, on='word', how='right').compute()
result[result[u'word'] == u'leonid']
```


    ---------------------------------------------------------------------------

    UnicodeEncodeError                        Traceback (most recent call last)

    <ipython-input-8-a8196fb6e4ca> in <module>()
    ----> 1 result = dask_df.merge(features, on='word', how='right').compute()
          2 result[result[u'word'] == u'leonid']
    

    C:\Anaconda2\lib\site-packages\dask\dataframe\core.pyc in merge(self, right, how, on, left_on, right_on, left_index, right_index, suffixes, indicator, npartitions, shuffle)
       2378                      right_index=right_index, suffixes=suffixes,
       2379                      npartitions=npartitions, indicator=indicator,
    -> 2380                      shuffle=shuffle)
       2381 
       2382     @derived_from(pd.DataFrame)
    

    C:\Anaconda2\lib\site-packages\dask\dataframe\multi.pyc in merge(left, right, how, on, left_on, right_on, left_index, right_index, suffixes, indicator, npartitions, shuffle, max_branch)
        394             right_on = False
        395             right_index = True
    --> 396         right = from_pandas(right, npartitions=1)  # turn into DataFrame
        397 
        398     # Both sides are now dd.DataFrame or dd.Series objects
    

    C:\Anaconda2\lib\site-packages\dask\dataframe\io\io.pyc in from_pandas(data, npartitions, chunksize, sort, name)
        180         npartitions = int(ceil(nrows / chunksize))
        181 
    --> 182     name = name or ('from_pandas-' + tokenize(data, chunksize))
        183 
        184     if not nrows:
    

    C:\Anaconda2\lib\site-packages\dask\base.pyc in tokenize(*args, **kwargs)
        366     if kwargs:
        367         args = args + (kwargs,)
    --> 368     return md5(str(tuple(map(normalize_token, args))).encode()).hexdigest()
    

    C:\Anaconda2\lib\site-packages\dask\utils.pyc in __call__(self, arg)
        547             pass
        548         else:
    --> 549             return impl(arg)
        550         # Is a lazy registration function present?
        551         toplevel, _, _ = typ.__module__.partition('.')
    

    C:\Anaconda2\lib\site-packages\dask\base.pyc in normalize_dataframe(df)
        320         data = [block.values for block in df._data.blocks]
        321         data += [df.columns, df.index]
    --> 322         return list(map(normalize_token, data))
        323 
        324 
    

    C:\Anaconda2\lib\site-packages\dask\utils.pyc in __call__(self, arg)
        547             pass
        548         else:
    --> 549             return impl(arg)
        550         # Is a lazy registration function present?
        551         toplevel, _, _ = typ.__module__.partition('.')
    

    C:\Anaconda2\lib\site-packages\dask\base.pyc in normalize_array(x)
        343                 data = md5('-'.join(x.flat).encode('utf-8')).hexdigest()
        344             except TypeError:
    --> 345                 data = md5(b'-'.join([str(item).encode() for item in x.flat])).hexdigest()
        346         else:
        347             try:
    

    UnicodeEncodeError: 'ascii' codec can't encode character u'\xfc' in position 1: ordinal not in range(128)


# Use smaller subset of data, same result 


```python
dask_df = dd.read_csv('./fastText_sample.csv', encoding='utf-8', usecols=list(range(1, 302)))
pd_df = pd.read_csv('./fastText_sample.csv', encoding='utf-8', usecols=list(range(1, 302)))
features = pd.read_csv('./features.csv', encoding='utf-8', usecols=[1])
```


```python
result = pd_df.merge(features, on='word', how='right')
result[result['word'] == 'leonid']
```




<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>word</th>
      <th>0</th>
      <th>1</th>
      <th>2</th>
      <th>3</th>
      <th>4</th>
      <th>5</th>
      <th>6</th>
      <th>7</th>
      <th>8</th>
      <th>...</th>
      <th>290</th>
      <th>291</th>
      <th>292</th>
      <th>293</th>
      <th>294</th>
      <th>295</th>
      <th>296</th>
      <th>297</th>
      <th>298</th>
      <th>299</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>810</th>
      <td>leonid</td>
      <td>-0.11284</td>
      <td>0.27493</td>
      <td>-0.11428</td>
      <td>0.27465</td>
      <td>-0.1484</td>
      <td>-0.16222</td>
      <td>0.54122</td>
      <td>-0.35387</td>
      <td>-0.44765</td>
      <td>...</td>
      <td>-0.35554</td>
      <td>-0.21266</td>
      <td>0.048219</td>
      <td>-0.023976</td>
      <td>0.23689</td>
      <td>-0.11721</td>
      <td>0.012078</td>
      <td>-0.15143</td>
      <td>-0.19005</td>
      <td>0.043254</td>
    </tr>
  </tbody>
</table>
<p>1 rows × 301 columns</p>
</div>




```python
result = dask_df.merge(features, on='word', how='right').compute()
result[result['word'] == 'leonid']
```


    ---------------------------------------------------------------------------

    UnicodeEncodeError                        Traceback (most recent call last)

    <ipython-input-25-74d8e0271756> in <module>()
    ----> 1 result = dask_df.merge(features, on='word', how='right').compute()
          2 result[result['word'] == 'leonid']
    

    C:\Anaconda2\lib\site-packages\dask\dataframe\core.pyc in merge(self, right, how, on, left_on, right_on, left_index, right_index, suffixes, indicator, npartitions, shuffle)
       2378                      right_index=right_index, suffixes=suffixes,
       2379                      npartitions=npartitions, indicator=indicator,
    -> 2380                      shuffle=shuffle)
       2381 
       2382     @derived_from(pd.DataFrame)
    

    C:\Anaconda2\lib\site-packages\dask\dataframe\multi.pyc in merge(left, right, how, on, left_on, right_on, left_index, right_index, suffixes, indicator, npartitions, shuffle, max_branch)
        394             right_on = False
        395             right_index = True
    --> 396         right = from_pandas(right, npartitions=1)  # turn into DataFrame
        397 
        398     # Both sides are now dd.DataFrame or dd.Series objects
    

    C:\Anaconda2\lib\site-packages\dask\dataframe\io\io.pyc in from_pandas(data, npartitions, chunksize, sort, name)
        180         npartitions = int(ceil(nrows / chunksize))
        181 
    --> 182     name = name or ('from_pandas-' + tokenize(data, chunksize))
        183 
        184     if not nrows:
    

    C:\Anaconda2\lib\site-packages\dask\base.pyc in tokenize(*args, **kwargs)
        366     if kwargs:
        367         args = args + (kwargs,)
    --> 368     return md5(str(tuple(map(normalize_token, args))).encode()).hexdigest()
    

    C:\Anaconda2\lib\site-packages\dask\utils.pyc in __call__(self, arg)
        547             pass
        548         else:
    --> 549             return impl(arg)
        550         # Is a lazy registration function present?
        551         toplevel, _, _ = typ.__module__.partition('.')
    

    C:\Anaconda2\lib\site-packages\dask\base.pyc in normalize_dataframe(df)
        320         data = [block.values for block in df._data.blocks]
        321         data += [df.columns, df.index]
    --> 322         return list(map(normalize_token, data))
        323 
        324 
    

    C:\Anaconda2\lib\site-packages\dask\utils.pyc in __call__(self, arg)
        547             pass
        548         else:
    --> 549             return impl(arg)
        550         # Is a lazy registration function present?
        551         toplevel, _, _ = typ.__module__.partition('.')
    

    C:\Anaconda2\lib\site-packages\dask\base.pyc in normalize_array(x)
        343                 data = md5('-'.join(x.flat).encode('utf-8')).hexdigest()
        344             except TypeError:
    --> 345                 data = md5(b'-'.join([str(item).encode() for item in x.flat])).hexdigest()
        346         else:
        347             try:
    

    UnicodeEncodeError: 'ascii' codec can't encode character u'\xfc' in position 1: ordinal not in range(128)


Thanks for the bug report, I've tracked this down to a bug in how we hash pandas object arrays (specifically object arrays that have missing values). Fix coming soon.

Resolving patch:
diff --git a/dask/dataframe/hashing.py b/dask/dataframe/hashing.py
--- a/dask/dataframe/hashing.py
+++ b/dask/dataframe/hashing.py
@@ -24,7 +24,7 @@
 # XXX: Pandas uses release branches > 0.19.0, which doesn't play well with
 # versioneer, since the release tags aren't ancestors of master. As such, we
 # need to use this hacky awfulness to check if we're > 0.19.2.
-if PANDAS_VERSION not in ('0.19.1', '0.19.2') and PANDAS_VERSION > '0.19.0+340':
+if PANDAS_VERSION not in ('0.19.1', '0.19.2') and PANDAS_VERSION > '0.19.0+460':
     from pandas.tools.hashing import hash_pandas_object
 else:
     from pandas.types.common import (is_categorical_dtype, is_numeric_dtype,
@@ -84,9 +84,18 @@ def adder(h, hashed_to_add):
         return h
 
     def _hash_categorical(c, encoding, hash_key):
-        cat_hashed = hash_array(c.categories.values, encoding, hash_key,
-                                categorize=False).astype(np.uint64, copy=False)
-        return c.rename_categories(cat_hashed).astype(np.uint64)
+        hashed = hash_array(c.categories.values, encoding, hash_key,
+                            categorize=False)
+        mask = c.isnull()
+        if len(hashed):
+            result = hashed.take(c.codes)
+        else:
+            result = np.zeros(len(mask), dtype='uint64')
+
+        if mask.any():
+            result[mask] = np.iinfo(np.uint64).max
+
+        return result
 
     def hash_array(vals, encoding='utf8', hash_key=None, categorize=True):
         if hash_key is None:


Test patch:
diff --git a/dask/dataframe/tests/test_hashing.py b/dask/dataframe/tests/test_hashing.py
--- a/dask/dataframe/tests/test_hashing.py
+++ b/dask/dataframe/tests/test_hashing.py
@@ -46,3 +46,12 @@ def test_categorical_consistency():
             h3 = hash_pandas_object(s3, categorize=categorize)
             tm.assert_series_equal(h1, h2)
             tm.assert_series_equal(h1, h3)
+
+
+def test_object_missing_values():
+    # Check that the presence of missing values doesn't change how object dtype
+    # is hashed.
+    s = pd.Series(['a', 'b', 'c', None])
+    h1 = hash_pandas_object(s).iloc[:3]
+    h2 = hash_pandas_object(s.iloc[:3])
+    tm.assert_series_equal(h1, h2)