Repository: dask/dask

Issue:
to_delayed always optimizes
The `da.Array.to_delayed` call optimizes the underlying dask array's task graph before creating the dask.delayed objects.  In some cases this is not desired.  One option here would be to have an `optimize_graph=True` keyword argument.  There might also be broader solutions?


Discussion:
Adding a flag seems like a good option to me, with the default of `True`.

> In some cases this is not desired.

For documentation, can you give an example?
When you have multiple arrays that share a common history.  Per-array optimization would remove this common history, generating redundant work.

```pyhon
x = da.from_array(...)
y = x + 1 + 2
z = 2 * x
ys = y.to_delayed() 
zs = z.to_delayed()
...
```
Perhaps we should have a toplevel `to_delayed` function?

```
yz, zs = da.to_delayed(ys, zs)
```

Resolving patch:
diff --git a/dask/array/core.py b/dask/array/core.py
--- a/dask/array/core.py
+++ b/dask/array/core.py
@@ -1833,10 +1833,14 @@ def __deepcopy__(self, memo):
         memo[id(self)] = c
         return c
 
-    def to_delayed(self):
-        """ Convert Array into dask Delayed objects
+    def to_delayed(self, optimize_graph=True):
+        """Convert into an array of ``dask.delayed`` objects, one per chunk.
 
-        Returns an array of values, one value per chunk.
+        Parameters
+        ----------
+        optimize_graph : bool, optional
+            If True [default], the graph is optimized before converting into
+            ``dask.delayed`` objects.
 
         See Also
         --------
@@ -1844,7 +1848,9 @@ def to_delayed(self):
         """
         from ..delayed import Delayed
         keys = self.__dask_keys__()
-        dsk = self.__dask_optimize__(self.__dask_graph__(), keys)
+        dsk = self.__dask_graph__()
+        if optimize_graph:
+            dsk = self.__dask_optimize__(dsk, keys)
         L = ndeepmap(self.ndim, lambda k: Delayed(k, dsk), keys)
         return np.array(L, dtype=object)
 
diff --git a/dask/bag/core.py b/dask/bag/core.py
--- a/dask/bag/core.py
+++ b/dask/bag/core.py
@@ -339,14 +339,19 @@ def apply(self, func):
 
     __int__ = __float__ = __complex__ = __bool__ = Base.compute
 
-    def to_delayed(self):
-        """ Convert bag item to dask.delayed.
+    def to_delayed(self, optimize_graph=True):
+        """Convert into a ``dask.delayed`` object.
 
-        Returns a single value.
+        Parameters
+        ----------
+        optimize_graph : bool, optional
+            If True [default], the graph is optimized before converting into
+            ``dask.delayed`` objects.
         """
         from dask.delayed import Delayed
-        dsk = self.__dask_optimize__(self.__dask_graph__(),
-                                     self.__dask_keys__())
+        dsk = self.__dask_graph__()
+        if optimize_graph:
+            dsk = self.__dask_optimize__(dsk, self.__dask_keys__())
         return Delayed(self.key, dsk)
 
 
@@ -1260,14 +1265,24 @@ def to_dataframe(self, meta=None, columns=None):
         divisions = [None] * (self.npartitions + 1)
         return dd.DataFrame(dsk, name, meta, divisions)
 
-    def to_delayed(self):
-        """ Convert bag to list of dask Delayed.
+    def to_delayed(self, optimize_graph=True):
+        """Convert into a list of ``dask.delayed`` objects, one per partition.
 
-        Returns list of Delayed, one per partition.
+        Parameters
+        ----------
+        optimize_graph : bool, optional
+            If True [default], the graph is optimized before converting into
+            ``dask.delayed`` objects.
+
+        See Also
+        --------
+        dask.bag.from_delayed
         """
         from dask.delayed import Delayed
         keys = self.__dask_keys__()
-        dsk = self.__dask_optimize__(self.__dask_graph__(), keys)
+        dsk = self.__dask_graph__()
+        if optimize_graph:
+            dsk = self.__dask_optimize__(dsk, keys)
         return [Delayed(k, dsk) for k in keys]
 
     def repartition(self, npartitions):
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -20,7 +20,7 @@
 from .. import core
 from ..utils import partial_by_order
 from .. import threaded
-from ..compatibility import apply, operator_div, bind_method, PY3
+from ..compatibility import apply, operator_div, bind_method
 from ..context import globalmethod
 from ..utils import (random_state_data, pseudorandom, derived_from, funcname,
                      memory_repr, put_lines, M, key_split, OperatorMethodMixin)
@@ -179,6 +179,21 @@ def f(self):
     def _get_binary_operator(cls, op, inv=False):
         return lambda self, other: _scalar_binary(op, self, other, inv=inv)
 
+    def to_delayed(self, optimize_graph=True):
+        """Convert into a ``dask.delayed`` object.
+
+        Parameters
+        ----------
+        optimize_graph : bool, optional
+            If True [default], the graph is optimized before converting into
+            ``dask.delayed`` objects.
+        """
+        from dask.delayed import Delayed
+        dsk = self.__dask_graph__()
+        if optimize_graph:
+            dsk = self.__dask_optimize__(dsk, self.__dask_keys__())
+        return Delayed(self.key, dsk)
+
 
 def _scalar_binary(op, self, other, inv=False):
     name = '{0}-{1}'.format(funcname(op), tokenize(self, other))
@@ -1014,9 +1029,29 @@ def to_csv(self, filename, **kwargs):
         from .io import to_csv
         return to_csv(self, filename, **kwargs)
 
-    def to_delayed(self):
-        """ See dd.to_delayed docstring for more information """
-        return to_delayed(self)
+    def to_delayed(self, optimize_graph=True):
+        """Convert into a list of ``dask.delayed`` objects, one per partition.
+
+        Parameters
+        ----------
+        optimize_graph : bool, optional
+            If True [default], the graph is optimized before converting into
+            ``dask.delayed`` objects.
+
+        Examples
+        --------
+        >>> partitions = df.to_delayed()  # doctest: +SKIP
+
+        See Also
+        --------
+        dask.dataframe.from_delayed
+        """
+        from dask.delayed import Delayed
+        keys = self.__dask_keys__()
+        dsk = self.__dask_graph__()
+        if optimize_graph:
+            dsk = self.__dask_optimize__(dsk, keys)
+        return [Delayed(k, dsk) for k in keys]
 
     @classmethod
     def _get_unary_operator(cls, op):
@@ -4032,19 +4067,24 @@ def maybe_shift_divisions(df, periods, freq):
     return df
 
 
-def to_delayed(df):
-    """ Create Dask Delayed objects from a Dask Dataframe
+def to_delayed(df, optimize_graph=True):
+    """Convert into a list of ``dask.delayed`` objects, one per partition.
 
-    Returns a list of delayed values, one value per partition.
+    Deprecated, please use the equivalent ``df.to_delayed`` method instead.
 
-    Examples
+    Parameters
+    ----------
+    optimize_graph : bool, optional
+        If True [default], the graph is optimized before converting into
+        ``dask.delayed`` objects.
+
+    See Also
     --------
-    >>> partitions = df.to_delayed()  # doctest: +SKIP
+    dask.dataframe.from_delayed
     """
-    from dask.delayed import Delayed
-    keys = df.__dask_keys__()
-    dsk = df.__dask_optimize__(df.__dask_graph__(), keys)
-    return [Delayed(k, dsk) for k in keys]
+    warnings.warn("DeprecationWarning: The `dd.to_delayed` function is "
+                  "deprecated, please use the `.to_delayed()` method instead.")
+    return df.to_delayed(optimize_graph=optimize_graph)
 
 
 @wraps(pd.to_datetime)
@@ -4071,7 +4111,3 @@ def _repr_data_series(s, index):
     else:
         dtype = str(s.dtype)
     return pd.Series([dtype] + ['...'] * npartitions, index=index, name=s.name)
-
-
-if PY3:
-    _Frame.to_delayed.__doc__ = to_delayed.__doc__


Test patch:
diff --git a/dask/array/tests/test_array_core.py b/dask/array/tests/test_array_core.py
--- a/dask/array/tests/test_array_core.py
+++ b/dask/array/tests/test_array_core.py
@@ -2370,12 +2370,20 @@ def test_to_delayed():
     assert a.compute() == s
 
 
-def test_to_delayed_optimizes():
+def test_to_delayed_optimize_graph():
     x = da.ones((4, 4), chunks=(2, 2))
     y = x[1:][1:][1:][:, 1:][:, 1:][:, 1:]
+
+    # optimizations
     d = y.to_delayed().flatten().tolist()[0]
     assert len([k for k in d.dask if k[0].startswith('getitem')]) == 1
 
+    # no optimizations
+    d2 = y.to_delayed(optimize_graph=False).flatten().tolist()[0]
+    assert dict(d2.dask) == dict(y.dask)
+
+    assert (d.compute() == d2.compute()).all()
+
 
 def test_cumulative():
     x = da.arange(20, chunks=5)
diff --git a/dask/bag/tests/test_bag.py b/dask/bag/tests/test_bag.py
--- a/dask/bag/tests/test_bag.py
+++ b/dask/bag/tests/test_bag.py
@@ -18,6 +18,7 @@
                            reduceby, reify, partition, inline_singleton_lists,
                            optimize, from_delayed)
 from dask.compatibility import BZ2File, GzipFile, PY2
+from dask.delayed import Delayed
 from dask.utils import filetexts, tmpfile, tmpdir
 from dask.utils_test import inc, add
 
@@ -972,8 +973,6 @@ def test_bag_compute_forward_kwargs():
 
 
 def test_to_delayed():
-    from dask.delayed import Delayed
-
     b = db.from_sequence([1, 2, 3, 4, 5, 6], npartitions=3)
     a, b, c = b.map(inc).to_delayed()
     assert all(isinstance(x, Delayed) for x in [a, b, c])
@@ -985,17 +984,24 @@ def test_to_delayed():
     assert t.compute() == 21
 
 
-def test_to_delayed_optimizes():
+def test_to_delayed_optimize_graph():
     b = db.from_sequence([1, 2, 3, 4, 5, 6], npartitions=1)
     b2 = b.map(inc).map(inc).map(inc)
 
     [d] = b2.to_delayed()
     text = str(dict(d.dask))
     assert text.count('reify') == 1
+    [d2] = b2.to_delayed(optimize_graph=False)
+    assert dict(d2.dask) == dict(b2.dask)
+    assert d.compute() == d2.compute()
 
-    d = b2.sum().to_delayed()
+    x = b2.sum()
+    d = x.to_delayed()
     text = str(dict(d.dask))
     assert text.count('reify') == 0
+    d2 = x.to_delayed(optimize_graph=False)
+    assert dict(d2.dask) == dict(x.dask)
+    assert d.compute() == d2.compute()
 
     [d] = b2.to_textfiles('foo.txt', compute=False)
     text = str(dict(d.dask))
diff --git a/dask/dataframe/io/tests/test_io.py b/dask/dataframe/io/tests/test_io.py
--- a/dask/dataframe/io/tests/test_io.py
+++ b/dask/dataframe/io/tests/test_io.py
@@ -528,17 +528,36 @@ def test_from_delayed_sorted():
 def test_to_delayed():
     df = pd.DataFrame({'x': [1, 2, 3, 4], 'y': [10, 20, 30, 40]})
     ddf = dd.from_pandas(df, npartitions=2)
+
+    # Frame
     a, b = ddf.to_delayed()
     assert isinstance(a, Delayed)
     assert isinstance(b, Delayed)
-
     assert_eq(a.compute(), df.iloc[:2])
 
+    # Scalar
+    x = ddf.x.sum()
+    dx = x.to_delayed()
+    assert isinstance(dx, Delayed)
+    assert_eq(dx.compute(), x)
+
 
-def test_to_delayed_optimizes():
+def test_to_delayed_optimize_graph():
     df = pd.DataFrame({'x': list(range(20))})
     ddf = dd.from_pandas(df, npartitions=20)
-    x = (ddf + 1).loc[:2]
+    ddf2 = (ddf + 1).loc[:2]
 
-    d = x.to_delayed()[0]
+    # Frame
+    d = ddf2.to_delayed()[0]
     assert len(d.dask) < 20
+    d2 = ddf2.to_delayed(optimize_graph=False)[0]
+    assert sorted(d2.dask) == sorted(ddf2.dask)
+    assert_eq(ddf2.get_partition(0), d.compute())
+    assert_eq(ddf2.get_partition(0), d2.compute())
+
+    # Scalar
+    x = ddf2.x.sum()
+    dx = x.to_delayed()
+    dx2 = x.to_delayed(optimize_graph=False)
+    assert len(dx.dask) < len(dx2.dask)
+    assert_eq(dx.compute(), dx2.compute())