Repository: dask/dask

Issue:
`dask.dataframe.utils.check_meta` raises in `dd.from_delayed` call with `string` dtype
```python
from dask import delayed
import dask.dataframe as dd
import pandas as pd

df = pd.DataFrame({"x": ["a"]}, dtype="object")
meta = pd.DataFrame({"x": []}, dtype="string")
task = delayed(df)

ddf = dd.from_delayed(task, meta)
ddf.compute(scheduler="single-threaded")
```

raises
```
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-2-7175373767c8> in <module>
      8
      9 ddf = dd.from_delayed(task, meta)
---> 10 ddf.compute(scheduler="single-threaded")

/opt/conda/envs/sd-pricing-r/lib/python3.8/site-packages/dask/base.py in compute(self, **kwargs)
    283         dask.base.compute
    284         """
--> 285         (result,) = compute(self, traverse=False, **kwargs)
    286         return result
    287

/opt/conda/envs/sd-pricing-r/lib/python3.8/site-packages/dask/base.py in compute(*args, **kwargs)
    565         postcomputes.append(x.__dask_postcompute__())
    566
--> 567     results = schedule(dsk, keys, **kwargs)
    568     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    569

/opt/conda/envs/sd-pricing-r/lib/python3.8/site-packages/dask/local.py in get_sync(dsk, keys, **kwargs)
    558     """
    559     kwargs.pop("num_workers", None)  # if num_workers present, remove it
--> 560     return get_async(
    561         synchronous_executor.submit,
    562         synchronous_executor._max_workers,

/opt/conda/envs/sd-pricing-r/lib/python3.8/site-packages/dask/local.py in get_async(submit, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, l
oads, chunksize, **kwargs)
    501             while state["waiting"] or state["ready"] or state["running"]:
    502                 fire_tasks(chunksize)
--> 503                 for key, res_info, failed in queue_get(queue).result():
    504                     if failed:
    505                         exc, tb = loads(res_info)

/opt/conda/envs/sd-pricing-r/lib/python3.8/concurrent/futures/_base.py in result(self, timeout)
    435                     raise CancelledError()
    436                 elif self._state == FINISHED:
--> 437                     return self.__get_result()
    438
    439                 self._condition.wait(timeout)

/opt/conda/envs/sd-pricing-r/lib/python3.8/concurrent/futures/_base.py in __get_result(self)
    387         if self._exception:
    388             try:
--> 389                 raise self._exception
    390             finally:
    391                 # Break a reference cycle with the exception in self._exception

/opt/conda/envs/sd-pricing-r/lib/python3.8/site-packages/dask/local.py in submit(self, fn, *args, **kwargs)
    543         fut = Future()
    544         try:
--> 545             fut.set_result(fn(*args, **kwargs))
    546         except BaseException as e:
    547             fut.set_exception(e)

/opt/conda/envs/sd-pricing-r/lib/python3.8/site-packages/dask/local.py in batch_execute_tasks(it)
    235     Batch computing of multiple tasks with `execute_task`
    236     """
--> 237     return [execute_task(*a) for a in it]
    238
    239
                                                                                                                                                                                                                   /opt/conda/envs/sd-pricing-r/lib/python3.8/site-packages/dask/local.py in <listcomp>(.0)
    235     Batch computing of multiple tasks with `execute_task`
    236     """
--> 237     return [execute_task(*a) for a in it]
    238
    239

/opt/conda/envs/sd-pricing-r/lib/python3.8/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    226         failed = False
    227     except BaseException as e:
--> 228         result = pack_exception(e, dumps)
    229         failed = True
    230     return key, result, failed

/opt/conda/envs/sd-pricing-r/lib/python3.8/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    221     try:
    222         task, data = loads(task_info)
--> 223         result = _execute_task(task, data)
    224         id = get_id()
    225         result = dumps((result, id))

/opt/conda/envs/sd-pricing-r/lib/python3.8/site-packages/dask/core.py in _execute_task(arg, cache, dsk)
    119         # temporaries by their reference count and can execute certain
    120         # operations in-place.
--> 121         return func(*(_execute_task(a, cache) for a in args))
    122     elif not ishashable(arg):
    123         return arg

/opt/conda/envs/sd-pricing-r/lib/python3.8/site-packages/dask/dataframe/utils.py in check_meta(x, meta, funcname, numeric_equal)
    656     elif is_dataframe_like(meta):
    657         dtypes = pd.concat([x.dtypes, meta.dtypes], axis=1, sort=True)
--> 658         bad_dtypes = [
    659             (repr(col), a, b)
    660             for col, a, b in dtypes.fillna("-").itertuples()

/opt/conda/envs/sd-pricing-r/lib/python3.8/site-packages/dask/dataframe/utils.py in <listcomp>(.0)
    659             (repr(col), a, b)
    660             for col, a, b in dtypes.fillna("-").itertuples()                                                                                                                                               --> 661             if not equal_dtypes(a, b)
    662         ]
    663         if bad_dtypes:

/opt/conda/envs/sd-pricing-r/lib/python3.8/site-packages/dask/dataframe/utils.py in equal_dtypes(a, b)
    637                 return True
    638             return a == b
--> 639         return (a.kind in eq_types and b.kind in eq_types) or (a == b)
    640
    641     if not (

TypeError: Cannot interpret 'StringDtype' as a data type
```

In `/opt/conda/envs/sd-pricing-r/lib/python3.8/site-packages/dask/dataframe/utils.py:639`, `a` is the `numpy.dtype` `dtype("O")` whereas `b` is a `pandas.core.arrays.string_.StringDtype`.

Note that the following works:
```
In [2]: df = pd.DataFrame({"x": [1]}, dtype="int64")
   ...: meta = pd.DataFrame({"x": []}, dtype="float64")
   ...: task = delayed(df)
   ...:
   ...: ddf = dd.from_delayed(task, meta)
   ...: ddf.compute().dtypes
Out[2]:
x    int64
dtype: object
```

I am not sure what the correct behaviour would be here.

dask: `2021.05.0`, pandas: `1.2.4`.



Discussion:
Thanks for raising this. I actually think that dask is behaving as expected here. Passing a `meta` does not coerce the dataframe to that dtype, it only lets dask know what to expect. 

So you have two options here:

Don't verify that the meta matches the output

```python
from dask import delayed
import dask.dataframe as dd
import pandas as pd

df = pd.DataFrame({"x": ["a"]}, dtype="object")
meta = pd.DataFrame({"x": []}, dtype="string")
task = delayed(df)

ddf = dd.from_delayed(task, meta, verify_meta=False)
output = ddf.compute(scheduler="single-threaded")
output.x.dtype
# dtype('O')
```

OR make sure that the dtype of the dataframe matches the meta

```python
from dask import delayed
import dask.dataframe as dd
import pandas as pd

df = pd.DataFrame({"x": ["a"]}, dtype="object")
df["x"] = df.x.astype("string")
meta = pd.DataFrame({"x": []}, dtype="string")
task = delayed(df)

ddf = dd.from_delayed(task, meta)
output = ddf.compute(scheduler="single-threaded")
output.x.dtype
# StringDtype
I agree. But shouldn't `dask.dataframe.utils.check_meta` return `False` here instead of raising?
Yeah you are right. I'm opening a PR now.

Resolving patch:
diff --git a/dask/dataframe/utils.py b/dask/dataframe/utils.py
--- a/dask/dataframe/utils.py
+++ b/dask/dataframe/utils.py
@@ -10,7 +10,7 @@
 import numpy as np
 import pandas as pd
 from pandas.api.types import is_scalar  # noqa: F401
-from pandas.api.types import is_categorical_dtype
+from pandas.api.types import is_categorical_dtype, is_dtype_equal
 
 from ..base import is_dask_collection
 from ..core import get_deps
@@ -363,7 +363,7 @@ def equal_dtypes(a, b):
             if UNKNOWN_CATEGORIES in a.categories or UNKNOWN_CATEGORIES in b.categories:
                 return True
             return a == b
-        return (a.kind in eq_types and b.kind in eq_types) or (a == b)
+        return (a.kind in eq_types and b.kind in eq_types) or is_dtype_equal(a, b)
 
     if not (
         is_dataframe_like(meta) or is_series_like(meta) or is_index_like(meta)


Test patch:
diff --git a/dask/dataframe/tests/test_utils_dataframe.py b/dask/dataframe/tests/test_utils_dataframe.py
--- a/dask/dataframe/tests/test_utils_dataframe.py
+++ b/dask/dataframe/tests/test_utils_dataframe.py
@@ -376,6 +376,22 @@ def test_check_meta():
     )
     assert str(err.value) == exp
 
+    if PANDAS_GT_100:
+        # pandas dtype metadata error
+        with pytest.raises(ValueError) as err:
+            check_meta(df.a, pd.Series([], dtype="string"), numeric_equal=False)
+        assert str(err.value) == (
+            "Metadata mismatch found.\n"
+            "\n"
+            "Partition type: `pandas.core.series.Series`\n"
+            "+----------+--------+\n"
+            "|          | dtype  |\n"
+            "+----------+--------+\n"
+            "| Found    | object |\n"
+            "| Expected | string |\n"
+            "+----------+--------+"
+        )
+
 
 def test_check_matching_columns_raises_appropriate_errors():
     df = pd.DataFrame(columns=["a", "b", "c"])