Repository: dask/dask

Issue:
dd.from_delayed ignores divisions keyword in dask 0.13
In dask 0.12:

```
print "dask version: %s" % str(dask.__version__)
pdf1 = pd.DataFrame(dict(A=[1], B = [2])).set_index('A')
pdf2 = pd.DataFrame(dict(A=[2], B = [2])).set_index('A')
ddf = dd.from_delayed([dask.delayed(pdf1), dask.delayed(pdf2)], divisions = [1,2,3])
ddf.known_divisions

>> dask version: 0.12.0
>> True
```

In dask 0.13:

```
print "dask version: %s" % str(dask.__version__)
pdf1 = pd.DataFrame(dict(A=[1], B = [2])).set_index('A')
pdf2 = pd.DataFrame(dict(A=[2], B = [2])).set_index('A')
ddf = dd.from_delayed([dask.delayed(pdf1), dask.delayed(pdf2)], divisions = [1,2,3])
ddf.known_divisions

>> dask version: 0.13.0
>> False
```


Discussion:
Verified.  That is indeed a problem.  Should be relatively simple to fix I
hope.

On Mon, Jan 9, 2017 at 4:45 AM, abast <notifications@github.com> wrote:

> In dask 0.12:
>
> print "dask version: %s" % str(dask.__version__)
> pdf1 = pd.DataFrame(dict(A=[1], B = [2])).set_index('A')
> pdf2 = pd.DataFrame(dict(A=[2], B = [2])).set_index('A')
> ddf = dd.from_delayed([dask.delayed(pdf1), dask.delayed(pdf2)], divisions = [1,2,3])
> ddf.known_divisions
>
> >> dask version: 0.12.0
> >> True
>
> In dask 0.13:
>
> print "dask version: %s" % str(dask.__version__)
> pdf1 = pd.DataFrame(dict(A=[1], B = [2])).set_index('A')
> pdf2 = pd.DataFrame(dict(A=[2], B = [2])).set_index('A')
> ddf = dd.from_delayed([dask.delayed(pdf1), dask.delayed(pdf2)], divisions = [1,2,3])
> ddf.known_divisions
>
> >> dask version: 0.13.0
> >> False
>
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/1897>, or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AASszNIb4IN6lQMg75eaNHsZqut1Sf5Hks5rQivRgaJpZM4LeOJp>
> .
>


Resolving patch:
diff --git a/dask/dataframe/io/io.py b/dask/dataframe/io/io.py
--- a/dask/dataframe/io/io.py
+++ b/dask/dataframe/io/io.py
@@ -562,7 +562,14 @@ def from_delayed(dfs, meta=None, divisions=None, prefix='from-delayed',
     else:
         Frame = DataFrame
 
-    df = Frame(dsk3, name, meta, [None] * (len(dfs) + 1))
+    if divisions is None or divisions == 'sorted':
+        divs = [None] * (len(dfs) + 1)
+    else:
+        divs = tuple(divisions)
+        if len(divs) != len(dfs) + 1:
+            raise ValueError("divisions should be a tuple of len(dfs) + 1")
+
+    df = Frame(dsk3, name, meta, divs)
 
     if divisions == 'sorted':
         from ..core import compute_divisions


Test patch:
diff --git a/dask/dataframe/io/tests/test_io.py b/dask/dataframe/io/tests/test_io.py
--- a/dask/dataframe/io/tests/test_io.py
+++ b/dask/dataframe/io/tests/test_io.py
@@ -11,7 +11,7 @@
 import dask.array as da
 import dask.dataframe as dd
 from dask.dataframe.io.io import _meta_from_array
-from dask.delayed import Delayed
+from dask.delayed import Delayed, delayed
 
 from dask.utils import tmpfile
 from dask.async import get_sync
@@ -555,3 +555,47 @@ def test_to_records():
     ddf = dd.from_pandas(df, 2)
 
     assert_eq(df.to_records(), ddf.to_records())
+
+
+def test_from_delayed():
+    df = pd.DataFrame(data=np.random.normal(size=(10, 4)), columns=list('abcd'))
+    parts = [df.iloc[:1], df.iloc[1:3], df.iloc[3:6], df.iloc[6:10]]
+    dfs = [delayed(parts.__getitem__)(i) for i in range(4)]
+    meta = dfs[0].compute()
+
+    my_len = lambda x: pd.Series([len(x)])
+
+    for divisions in [None, [0, 1, 3, 6, 10]]:
+        ddf = dd.from_delayed(dfs, meta=meta, divisions=divisions)
+        assert_eq(ddf, df)
+        assert list(ddf.map_partitions(my_len).compute()) == [1, 2, 3, 4]
+        assert ddf.known_divisions == (divisions is not None)
+
+        s = dd.from_delayed([d.a for d in dfs], meta=meta.a,
+                            divisions=divisions)
+        assert_eq(s, df.a)
+        assert list(s.map_partitions(my_len).compute()) == [1, 2, 3, 4]
+        assert ddf.known_divisions == (divisions is not None)
+
+    with pytest.raises(ValueError):
+        dd.from_delayed(dfs, meta=meta, divisions=[0, 1, 3, 6])
+
+
+def test_from_delayed_sorted():
+    a = pd.DataFrame({'x': [1, 2]}, index=[1, 10])
+    b = pd.DataFrame({'x': [4, 1]}, index=[100, 200])
+
+    A = dd.from_delayed([delayed(a), delayed(b)], divisions='sorted')
+    assert A.known_divisions
+
+    assert A.divisions == (1, 100, 200)
+
+
+def test_to_delayed():
+    df = pd.DataFrame({'x': [1, 2, 3, 4], 'y': [10, 20, 30, 40]})
+    ddf = dd.from_pandas(df, npartitions=2)
+    a, b = ddf.to_delayed()
+    assert isinstance(a, Delayed)
+    assert isinstance(b, Delayed)
+
+    assert_eq(a.compute(), df.iloc[:2])
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -2344,43 +2344,6 @@ def test_dataframe_itertuples():
         assert a == b
 
 
-def test_from_delayed():
-    dfs = [delayed(tm.makeTimeDataFrame)(i) for i in range(1, 5)]
-    meta = dfs[0].compute()
-    df = dd.from_delayed(dfs, meta=meta)
-
-    assert (df.compute().columns == df.columns).all()
-    f = lambda x: pd.Series([len(x)])
-    assert list(df.map_partitions(f).compute()) == [1, 2, 3, 4]
-
-    ss = [d.A for d in dfs]
-    s = dd.from_delayed(ss, meta=meta.A)
-
-    assert s.compute().name == s.name
-    assert list(s.map_partitions(f).compute()) == [1, 2, 3, 4]
-
-
-def test_from_delayed_sorted():
-    a = pd.DataFrame({'x': [1, 2]}, index=[1, 10])
-    b = pd.DataFrame({'x': [4, 1]}, index=[100, 200])
-
-    A = dd.from_delayed([delayed(a), delayed(b)], divisions='sorted')
-    assert A.known_divisions
-
-    assert A.divisions == (1, 100, 200)
-
-
-def test_to_delayed():
-    from dask.delayed import Delayed
-    df = pd.DataFrame({'x': [1, 2, 3, 4], 'y': [10, 20, 30, 40]})
-    ddf = dd.from_pandas(df, npartitions=2)
-    a, b = ddf.to_delayed()
-    assert isinstance(a, Delayed)
-    assert isinstance(b, Delayed)
-
-    assert_eq(a.compute(), df.iloc[:2])
-
-
 def test_astype():
     df = pd.DataFrame({'x': [1, 2, 3, None], 'y': [10, 20, 30, 40]},
                       index=[10, 20, 30, 40])