Repository: dask/dask

Issue:
Generalize `dd.to_datetime` to support cuDF-backed Series
A couple months ago I opened Issue https://github.com/rapidsai/cudf/issues/12062 on cuDF, but after some discussions with @wence- @beckernick @charlesbluca it seems like it can be resolved on the Dask side.

**Minimal Complete Verifiable Example**:
```
import cudf
import dask_cudf
import dask.dataframe as dd

a = cudf.Series(['2022-01-01', '2022-02-02'])
dser = dask_cudf.from_cudf(a, 1)
res = dd.to_datetime(dser).compute()  # note that there are no errors if we don't call compute()
```
```
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In [19], line 6
      4 a = cudf.Series(['2022-01-01', '2022-02-02'])
      5 dser = dask_cudf.from_cudf(a, 1)
----> 6 res = dd.to_datetime(dser).compute()

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/dask/base.py:315, in DaskMethodsMixin.compute(self, **kwargs)
    291 def compute(self, **kwargs):
    292     """Compute this dask collection
    293 
    294     This turns a lazy Dask collection into its in-memory equivalent.
   (...)
    313     dask.base.compute
    314     """
--> 315     (result,) = compute(self, traverse=False, **kwargs)
    316     return result

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/dask/base.py:600, in compute(traverse, optimize_graph, scheduler, get, *args, **kwargs)
    597     keys.append(x.__dask_keys__())
    598     postcomputes.append(x.__dask_postcompute__())
--> 600 results = schedule(dsk, keys, **kwargs)
    601 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/dask/threaded.py:89, in get(dsk, keys, cache, num_workers, pool, **kwargs)
     86     elif isinstance(pool, multiprocessing.pool.Pool):
     87         pool = MultiprocessingPoolExecutor(pool)
---> 89 results = get_async(
     90     pool.submit,
     91     pool._max_workers,
     92     dsk,
     93     keys,
     94     cache=cache,
     95     get_id=_thread_get_id,
     96     pack_exception=pack_exception,
     97     **kwargs,
     98 )
    100 # Cleanup pools associated to dead threads
    101 with pools_lock:

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/dask/local.py:511, in get_async(submit, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, chunksize, **kwargs)
    509         _execute_task(task, data)  # Re-execute locally
    510     else:
--> 511         raise_exception(exc, tb)
    512 res, worker_id = loads(res_info)
    513 state["cache"][key] = res

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/dask/local.py:319, in reraise(exc, tb)
    317 if exc.__traceback__ is not tb:
    318     raise exc.with_traceback(tb)
--> 319 raise exc

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/dask/local.py:224, in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    222 try:
    223     task, data = loads(task_info)
--> 224     result = _execute_task(task, data)
    225     id = get_id()
    226     result = dumps((result, id))

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/dask/core.py:119, in _execute_task(arg, cache, dsk)
    115     func, args = arg[0], arg[1:]
    116     # Note: Don't assign the subtask results to a variable. numpy detects
    117     # temporaries by their reference count and can execute certain
    118     # operations in-place.
--> 119     return func(*(_execute_task(a, cache) for a in args))
    120 elif not ishashable(arg):
    121     return arg

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/dask/optimization.py:990, in SubgraphCallable.__call__(self, *args)
    988 if not len(args) == len(self.inkeys):
    989     raise ValueError("Expected %d args, got %d" % (len(self.inkeys), len(args)))
--> 990 return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/dask/core.py:149, in get(dsk, out, cache)
    147 for key in toposort(dsk):
    148     task = dsk[key]
--> 149     result = _execute_task(task, cache)
    150     cache[key] = result
    151 result = _execute_task(out, cache)

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/dask/core.py:119, in _execute_task(arg, cache, dsk)
    115     func, args = arg[0], arg[1:]
    116     # Note: Don't assign the subtask results to a variable. numpy detects
    117     # temporaries by their reference count and can execute certain
    118     # operations in-place.
--> 119     return func(*(_execute_task(a, cache) for a in args))
    120 elif not ishashable(arg):
    121     return arg

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/dask/utils.py:71, in apply(func, args, kwargs)
     40 """Apply a function given its positional and keyword arguments.
     41 
     42 Equivalent to ``func(*args, **kwargs)``
   (...)
     68 >>> dsk = {'task-name': task}  # adds the task to a low level Dask task graph
     69 """
     70 if kwargs:
---> 71     return func(*args, **kwargs)
     72 else:
     73     return func(*args)

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/dask/dataframe/core.py:6770, in apply_and_enforce(*args, **kwargs)
   6768 func = kwargs.pop("_func")
   6769 meta = kwargs.pop("_meta")
-> 6770 df = func(*args, **kwargs)
   6771 if is_dataframe_like(df) or is_series_like(df) or is_index_like(df):
   6772     if not len(df):

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/pandas/core/tools/datetimes.py:1100, in to_datetime(arg, errors, dayfirst, yearfirst, utc, format, exact, unit, infer_datetime_format, origin, cache)
   1098         result = _convert_and_box_cache(argc, cache_array)
   1099     else:
-> 1100         result = convert_listlike(argc, format)
   1101 else:
   1102     result = convert_listlike(np.array([arg]), format)[0]

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/pandas/core/tools/datetimes.py:413, in _convert_listlike_datetimes(arg, format, name, tz, unit, errors, infer_datetime_format, dayfirst, yearfirst, exact)
    410         return idx
    411     raise
--> 413 arg = ensure_object(arg)
    414 require_iso8601 = False
    416 if infer_datetime_format and format is None:

File pandas/_libs/algos_common_helper.pxi:33, in pandas._libs.algos.ensure_object()

File ~/miniconda3/envs/dsql_rapids-22.12/lib/python3.9/site-packages/cudf/core/frame.py:447, in Frame.__array__(self, dtype)
    446 def __array__(self, dtype=None):
--> 447     raise TypeError(
    448         "Implicit conversion to a host NumPy array via __array__ is not "
    449         "allowed, To explicitly construct a GPU matrix, consider using "
    450         ".to_cupy()\nTo explicitly construct a host matrix, consider "
    451         "using .to_numpy()."
    452     )

TypeError: Implicit conversion to a host NumPy array via __array__ is not allowed, To explicitly construct a GPU matrix, consider using .to_cupy()
To explicitly construct a host matrix, consider using .to_numpy().
```

**Anything else we need to know?**:
@beckernick traced `to_datetime()` [here](https://github.com/dask/dask/blob/23a75632b7644f1289a760031d9ba85a35cbfd20/dask/dataframe/core.py#L7845) and suggested to generalize the Dask implementation by potentially using [this pattern](https://github.com/dask/dask/blob/23a75632b7644f1289a760031d9ba85a35cbfd20/dask/dataframe/reshape.py#L154-L155).



Discussion:

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -81,6 +81,7 @@
     apply,
     derived_from,
     funcname,
+    get_meta_library,
     has_keyword,
     is_arraylike,
     iter_chunks,
@@ -8037,7 +8038,7 @@ def to_datetime(arg, meta=None, **kwargs):
     tz_kwarg = {"tz": "utc"} if kwargs.get("utc") else {}
     if meta is None:
         if isinstance(arg, Index):
-            meta = pd.DatetimeIndex([], **tz_kwarg)
+            meta = get_meta_library(arg).DatetimeIndex([], **tz_kwarg)
             meta.name = arg.name
         elif not (is_dataframe_like(arg) or is_series_like(arg)):
             raise NotImplementedError(
@@ -8058,7 +8059,7 @@ def to_datetime(arg, meta=None, **kwargs):
         )
         kwargs.pop("infer_datetime_format")
 
-    return map_partitions(pd.to_datetime, arg, meta=meta, **kwargs)
+    return map_partitions(get_meta_library(arg).to_datetime, arg, meta=meta, **kwargs)
 
 
 @wraps(pd.to_timedelta)
diff --git a/dask/dataframe/reshape.py b/dask/dataframe/reshape.py
--- a/dask/dataframe/reshape.py
+++ b/dask/dataframe/reshape.py
@@ -1,5 +1,3 @@
-import sys
-
 import numpy as np
 import pandas as pd
 from pandas.api.types import is_list_like, is_scalar
@@ -9,7 +7,7 @@
 from dask.dataframe._compat import PANDAS_GT_200
 from dask.dataframe.core import DataFrame, Series, apply_concat_apply, map_partitions
 from dask.dataframe.utils import has_known_categories
-from dask.utils import M
+from dask.utils import M, get_meta_library
 
 ###############################################################
 # Dummies
@@ -154,11 +152,8 @@ def get_dummies(
         if not all(has_known_categories(data[c]) for c in columns):
             raise NotImplementedError(unknown_cat_msg)
 
-    package_name = data._meta.__class__.__module__.split(".")[0]
-    dummies = sys.modules[package_name].get_dummies
-
     return map_partitions(
-        dummies,
+        get_meta_library(data).get_dummies,
         data,
         prefix=prefix,
         prefix_sep=prefix_sep,
diff --git a/dask/utils.py b/dask/utils.py
--- a/dask/utils.py
+++ b/dask/utils.py
@@ -2104,3 +2104,10 @@ def get_default_shuffle_algorithm() -> str:
         return "tasks"
     except (ImportError, ValueError):
         return "disk"
+
+
+def get_meta_library(like):
+    if hasattr(like, "_meta"):
+        like = like._meta
+
+    return import_module(typename(like).partition(".")[0])


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -4709,15 +4709,24 @@ def test_datetime_loc_open_slicing():
     assert_eq(df[0].loc["02.02.2015":], ddf[0].loc["02.02.2015":])
 
 
-def test_to_datetime():
-    df = pd.DataFrame({"year": [2015, 2016], "month": [2, 3], "day": [4, 5]})
+@pytest.mark.parametrize("gpu", [False, pytest.param(True, marks=pytest.mark.gpu)])
+def test_to_datetime(gpu):
+    xd = pd if not gpu else pytest.importorskip("cudf")
+
+    # meta dtype is inconsistent for cuDF-backed frames
+    check_dtype = not gpu
+
+    df = xd.DataFrame({"year": [2015, 2016], "month": ["2", "3"], "day": [4, 5]})
     df.index.name = "ix"
     ddf = dd.from_pandas(df, npartitions=2)
 
-    assert_eq(pd.to_datetime(df), dd.to_datetime(ddf))
+    assert_eq(xd.to_datetime(df), dd.to_datetime(ddf), check_dtype=check_dtype)
+    assert_eq(xd.to_datetime(df), dd.to_datetime(df), check_dtype=check_dtype)
 
-    s = pd.Series(["3/11/2000", "3/12/2000", "3/13/2000"] * 100)
-    s.index = s.values
+    s = xd.Series(
+        ["3/11/2000", "3/12/2000", "3/13/2000"] * 100,
+        index=["3/11/2000", "3/12/2000", "3/13/2000"] * 100,
+    )
     ds = dd.from_pandas(s, npartitions=10, sort=False)
 
     if PANDAS_GT_200:
@@ -4726,23 +4735,32 @@ def test_to_datetime():
         ctx = contextlib.nullcontext()
 
     with ctx:
-        expected = pd.to_datetime(s, infer_datetime_format=True)
+        expected = xd.to_datetime(s, infer_datetime_format=True)
     with ctx:
         result = dd.to_datetime(ds, infer_datetime_format=True)
-    assert_eq(expected, result)
+    assert_eq(expected, result, check_dtype=check_dtype)
+    with ctx:
+        result = dd.to_datetime(s, infer_datetime_format=True)
+    assert_eq(expected, result, check_dtype=check_dtype)
 
     with ctx:
-        expected = pd.to_datetime(s.index, infer_datetime_format=True)
+        expected = xd.to_datetime(s.index, infer_datetime_format=True)
     with ctx:
         result = dd.to_datetime(ds.index, infer_datetime_format=True)
     assert_eq(expected, result, check_divisions=False)
 
-    assert_eq(
-        pd.to_datetime(s, utc=True),
-        dd.to_datetime(ds, utc=True),
-    )
+    # cuDF does not yet support timezone-aware datetimes
+    if not gpu:
+        assert_eq(
+            xd.to_datetime(s, utc=True),
+            dd.to_datetime(ds, utc=True),
+        )
+        assert_eq(
+            xd.to_datetime(s, utc=True),
+            dd.to_datetime(s, utc=True),
+        )
 
-    for arg in ("2021-08-03", 2021):
+    for arg in ("2021-08-03", 2021, s.index):
         with pytest.raises(NotImplementedError, match="non-index-able arguments"):
             dd.to_datetime(arg)
 
diff --git a/dask/tests/test_utils.py b/dask/tests/test_utils.py
--- a/dask/tests/test_utils.py
+++ b/dask/tests/test_utils.py
@@ -26,6 +26,7 @@
     format_bytes,
     format_time,
     funcname,
+    get_meta_library,
     getargspec,
     has_keyword,
     is_arraylike,
@@ -894,3 +895,41 @@ def test_tmpfile_naming():
     with tmpfile(extension=".jpg") as fn:
         assert fn[-4:] == ".jpg"
         assert fn[-5] != "."
+
+
+def test_get_meta_library():
+    np = pytest.importorskip("numpy")
+    pd = pytest.importorskip("pandas")
+    da = pytest.importorskip("dask.array")
+    dd = pytest.importorskip("dask.dataframe")
+
+    assert get_meta_library(pd.DataFrame()) == pd
+    assert get_meta_library(np.array([])) == np
+
+    assert get_meta_library(pd.DataFrame()) == get_meta_library(pd.DataFrame)
+    assert get_meta_library(np.ndarray([])) == get_meta_library(np.ndarray)
+
+    assert get_meta_library(pd.DataFrame()) == get_meta_library(
+        dd.from_dict({}, npartitions=1)
+    )
+    assert get_meta_library(np.ndarray([])) == get_meta_library(da.from_array([]))
+
+
+def test_get_meta_library_gpu():
+    cp = pytest.importorskip("cupy")
+    cudf = pytest.importorskip("cudf")
+    da = pytest.importorskip("dask.array")
+    dd = pytest.importorskip("dask.dataframe")
+
+    assert get_meta_library(cudf.DataFrame()) == cudf
+    assert get_meta_library(cp.array([])) == cp
+
+    assert get_meta_library(cudf.DataFrame()) == get_meta_library(cudf.DataFrame)
+    assert get_meta_library(cp.ndarray([])) == get_meta_library(cp.ndarray)
+
+    assert get_meta_library(cudf.DataFrame()) == get_meta_library(
+        dd.from_dict({}, npartitions=1).to_backend("cudf")
+    )
+    assert get_meta_library(cp.ndarray([])) == get_meta_library(
+        da.from_array([]).to_backend("cupy")
+    )