Repository: dask/dask

Issue:
Use repartition to increase the number of partitions
This works fine when the user supplies divisions but doesn't work otherwise.



Discussion:
Would be very useful! Is there any workaround for this limitation atm?
Tried this but doesn't work because the delayed returns a dask dataframe.
Maybe I could construct a dask.DataFrame from the delayed object manually?
```
fut = dd.from_delayed(delayed(dd.from_pandas)(big_intermediate, chunksize=2))
```
Would be super useful. I think, it worked some versions ago.

Resolving patch:
diff --git a/dask/bag/core.py b/dask/bag/core.py
--- a/dask/bag/core.py
+++ b/dask/bag/core.py
@@ -4,6 +4,7 @@
 from functools import wraps, partial
 import itertools
 import math
+from operator import getitem
 import os
 import types
 import uuid
@@ -1117,23 +1118,40 @@ def repartition(self, npartitions):
         --------
         >>> b.repartition(5)  # set to have 5 partitions  # doctest: +SKIP
         """
-        if npartitions > self.npartitions:
-            msg = ("Repartition only supports going to fewer partitions\n"
-                   " old: %d  new: %d")
-            raise NotImplementedError(msg % (self.npartitions, npartitions))
-        npartitions_ratio = self.npartitions / npartitions
-        new_partitions_boundaries = [int(old_partition_index * npartitions_ratio)
-                                     for old_partition_index in range(npartitions + 1)]
-        new_name = 'repartition-%d-%s' % (npartitions, tokenize(self))
-
-        dsk = {}
-        for new_partition_index in range(npartitions):
-            value = (list, (toolz.concat,
-                            [(self.name, old_partition_index)
-                             for old_partition_index in
-                             range(new_partitions_boundaries[new_partition_index],
-                                   new_partitions_boundaries[new_partition_index + 1])]))
-            dsk[new_name, new_partition_index] = value
+        new_name = 'repartition-%d-%s' % (npartitions, tokenize(self, npartitions))
+        if npartitions == self.npartitions:
+            return self
+        elif npartitions < self.npartitions:
+            ratio = self.npartitions / npartitions
+            new_partitions_boundaries = [int(old_partition_index * ratio)
+                                         for old_partition_index in range(npartitions + 1)]
+
+            dsk = {}
+            for new_partition_index in range(npartitions):
+                value = (list, (toolz.concat,
+                                [(self.name, old_partition_index)
+                                 for old_partition_index in
+                                 range(new_partitions_boundaries[new_partition_index],
+                                       new_partitions_boundaries[new_partition_index + 1])]))
+                dsk[new_name, new_partition_index] = value
+        else:  # npartitions > self.npartitions
+            ratio = npartitions / self.npartitions
+            split_name = 'split-%s' % tokenize(self, npartitions)
+            dsk = {}
+            last = 0
+            j = 0
+            for i in range(self.npartitions):
+                new = last + ratio
+                if i == self.npartitions - 1:
+                    k = npartitions - j
+                else:
+                    k = int(new - last)
+                dsk[(split_name, i)] = (split, (self.name, i), k)
+                for jj in range(k):
+                    dsk[(new_name, j)] = (getitem, (split_name, i), jj)
+                    j += 1
+                last = new
+
         return Bag(dsk=merge(self.dask, dsk), name=new_name, npartitions=npartitions)
 
     def accumulate(self, binop, initial=no_default):
@@ -1677,3 +1695,18 @@ def random_state_data_python(n, random_state=None):
     maxuint32 = 1 << 32
     return [tuple(random_state.randint(0, maxuint32) for i in range(624))
             for i in range(n)]
+
+
+def split(seq, n):
+    """ Split apart a sequence into n equal pieces
+
+    >>> split(range(10), 3)
+    [[0, 1, 2], [3, 4, 5], [6, 7, 8, 9]]
+    """
+    if not isinstance(seq, (list, tuple)):
+        seq = list(seq)
+
+    part = len(seq) / n
+    L = [seq[int(part * i): int(part * (i + 1))] for i in range(n - 1)]
+    L.append(seq[int(part * (n - 1)):])
+    return L


Test patch:
diff --git a/dask/bag/tests/test_bag.py b/dask/bag/tests/test_bag.py
--- a/dask/bag/tests/test_bag.py
+++ b/dask/bag/tests/test_bag.py
@@ -915,19 +915,29 @@ def test_zip(npartitions, hi=1000):
     assert list(pairs) == list(zip(range(0, hi, 2), range(1, hi, 2)))
 
 
-def test_repartition():
-    for x, y in [(10, 5), (7, 3), (5, 1), (5, 4)]:
-        b = db.from_sequence(range(20), npartitions=x)
-        c = b.repartition(y)
+@pytest.mark.parametrize('nin', [1, 2, 7, 11, 23])
+@pytest.mark.parametrize('nout', [1, 2, 5, 12, 23])
+def test_repartition(nin, nout):
+    b = db.from_sequence(range(100), npartitions=nin)
+    c = b.repartition(npartitions=nout)
 
-        assert b.npartitions == x
-        assert c.npartitions == y
-        assert list(b) == c.compute(get=dask.get)
+    assert c.npartitions == nout
+    assert b.compute(get=dask.get) == c.compute(get=dask.get)
+    results = dask.get(c.dask, c._keys())
+    assert all(results)
 
-    try:
-        b.repartition(100)
-    except NotImplementedError as e:
-        assert '100' in str(e)
+
+def test_repartition_names():
+    b = db.from_sequence(range(100), npartitions=5)
+    c = b.repartition(2)
+    assert b.name != c.name
+
+    d = b.repartition(20)
+    assert b.name != c.name
+    assert c.name != d.name
+
+    c = b.repartition(5)
+    assert b is c
 
 
 @pytest.mark.skipif('not db.core._implement_accumulate')