Repository: dask/dask

Issue:
Consistency in da.random
Currently multiple calls to `da.random` functions don't produce consistent values

``` python
In [1]: import dask.array as da

In [2]: x = da.random.random((2,), chunks=2)

In [3]: x.compute()
Out[3]: array([ 0.47001845,  0.54208932])

In [4]: x.compute()
Out[4]: array([ 0.23277611,  0.312902  ])
```

We should correct this, either by using `np.random.seed` in each block (perhaps with a lock around the pair of calls?) or with the RandomState API.

cc @shoyer @ogrisel



Discussion:
I would rather have an object oriented API to make it possible to seed the RNG for deterministic results akin to the `numpy.random.RandomState` API. E.g. to seed with the 42 integer which is arguably the best seed for random numbers:

``` python
x = da.random.RandomState(42).normal(size=(2,), chunks=2)
```

Can do.  I'll play with this on the plane today.


Resolving patch:
diff --git a/dask/array/random.py b/dask/array/random.py
--- a/dask/array/random.py
+++ b/dask/array/random.py
@@ -1,56 +1,314 @@
 from __future__ import absolute_import, division, print_function
 
 import numpy as np
-from .wrap import wrap, wrap_func_size_as_kwarg
+from itertools import product
+from toolz import curry
+from .core import normalize_chunks, Array, names
 
-"""
-Univariate distributions
-"""
+def doc_wraps(func):
+    """ Copy docstring from one function to another """
+    def _(func2):
+        func2.__doc__ = func.__doc__.replace('>>>', '>>').replace('...', '..')
+        return func2
+    return _
 
-wrap = wrap(wrap_func_size_as_kwarg)
-
-random = wrap(np.random.random)
-beta = wrap(np.random.beta)
-binomial = wrap(np.random.binomial)
-chisquare = wrap(np.random.chisquare)
-exponential = wrap(np.random.exponential)
-f = wrap(np.random.f)
-gamma = wrap(np.random.gamma)
-geometric = wrap(np.random.geometric)
-gumbel = wrap(np.random.gumbel)
-hypergeometric = wrap(np.random.hypergeometric)
-laplace = wrap(np.random.laplace)
-logistic = wrap(np.random.logistic)
-lognormal = wrap(np.random.lognormal)
-logseries = wrap(np.random.logseries)
-negative_binomial = wrap(np.random.negative_binomial)
-noncentral_chisquare = wrap(np.random.noncentral_chisquare)
-noncentral_f = wrap(np.random.noncentral_f)
-normal = wrap(np.random.normal)
-pareto = wrap(np.random.pareto)
-poisson = wrap(np.random.poisson)
-power = wrap(np.random.power)
-rayleigh = wrap(np.random.rayleigh)
-triangular = wrap(np.random.triangular)
-uniform = wrap(np.random.uniform)
-vonmises = wrap(np.random.vonmises)
-wald = wrap(np.random.wald)
-weibull = wrap(np.random.weibull)
-zipf = wrap(np.random.zipf)
 
-"""
-Standard distributions
-"""
+class RandomState(object):
+    """
+    Mersenne Twister pseudo-random number generator
 
-standard_cauchy = wrap(np.random.standard_cauchy)
-standard_exponential = wrap(np.random.standard_exponential)
-standard_gamma = wrap(np.random.standard_gamma)
-standard_normal = wrap(np.random.standard_normal)
-standard_t = wrap(np.random.standard_t)
+    This object contains state to deterministicly generate pseudo-random
+    numbers from a variety of probabilitiy distributions.  It is identical to
+    ``np.random.RandomState`` except that all functions also take a ``chunks=``
+    keyword argument.
 
-"""
-TODO: Multivariate distributions
+    Examples
+    --------
+
+    >>> import dask.array as da
+    >>> state = da.random.RandomState(1234)  # a seed
+    >>> x = state.normal(10, 0.1, size=3, chunks=(2,))
+    >>> x.compute()
+    array([  9.95487579,  10.02999135,  10.08498441])
+
+    See Also:
+        np.random.RandomState
+    """
+    def __init__(self, seed=None):
+        self._numpy_state = np.random.RandomState(seed)
+
+    def _wrap(self, func, *args, **kwargs):
+        size = kwargs.pop('size')
+        chunks = kwargs.pop('chunks')
+
+        if not isinstance(size, (tuple, list)):
+            size = (size,)
+
+        chunks = normalize_chunks(chunks, size)
+        name = next(names)
+
+        # Get dtype
+        kw = kwargs.copy()
+        kw['size'] = (0,)
+        dtype = func(np.random.RandomState(), *args, **kw).dtype
+
+        # Build graph
+        keys = product([name], *[range(len(bd)) for bd in chunks])
+        sizes = product(*chunks)
+        vals = ((_apply_random,
+                  func.__name__,
+                  self._numpy_state.randint(np.iinfo(np.uint32).max),
+                  size, args, kwargs)
+                for size in sizes)
+        dsk = dict(zip(keys, vals))
+
+        return Array(dsk, name, chunks, dtype=dtype)
+
+    @doc_wraps(np.random.RandomState.beta)
+    def beta(self, a, b, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.beta, a, b,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.binomial)
+    def binomial(self, n, p, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.binomial, n, p,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.chisquare)
+    def chisquare(self, df, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.chisquare, df,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.choice)
+    def choice(self, a, size=None, replace=True, p=None, chunks=None):
+        return self._wrap(np.random.RandomState.choice, a,
+                         size=size, replace=True, p=None, chunks=chunks)
+
+    # @doc_wraps(np.random.RandomState.dirichlet)
+    # def dirichlet(self, alpha, size=None, chunks=None):
+
+    @doc_wraps(np.random.RandomState.exponential)
+    def exponential(self, scale=1.0, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.exponential, scale,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.f)
+    def f(self, dfnum, dfden, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.f, dfnum, dfden,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.gamma)
+    def gamma(self, shape, scale=1.0, chunks=None):
+        return self._wrap(np.random.RandomState.gamma, scale,
+                         size=shape, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.geometric)
+    def geometric(self, p, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.geometric, p,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.gumbel)
+    def gumbel(self, loc=0.0, scale=1.0, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.gumbel, loc, scale,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.hypergeometric)
+    def hypergeometric(self, ngood, nbad, nsample, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.hypergeometric,
+                         ngood, nbad, nsample,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.laplace)
+    def laplace(self, loc=0.0, scale=1.0, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.laplace, loc, scale,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.logistic)
+    def logistic(self, loc=0.0, scale=1.0, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.logistic, loc, scale,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.lognormal)
+    def lognormal(self, mean=0.0, sigma=1.0, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.lognormal, mean, sigma,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.logseries)
+    def logseries(self, p, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.logseries, p,
+                         size=size, chunks=chunks)
+
+    # multinomial
+
+    @doc_wraps(np.random.RandomState.negative_binomial)
+    def negative_binomial(self, n, p, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.negative_binomial, n, p,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.noncentral_chisquare)
+    def noncentral_chisquare(self, df, nonc, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.noncentral_chisquare, df, nonc,
+                         size=size, chunks=chunks)
 
-dirichlet =
-multinomial =
+    @doc_wraps(np.random.RandomState.noncentral_f)
+    def noncentral_f(self, dfnum, dfden, nonc,  size=None, chunks=None):
+        return self._wrap(np.random.RandomState.noncentral_f,
+                         dfnum, dfden, nonc,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.normal)
+    def normal(self, loc=0.0, scale=1.0, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.normal, loc, scale,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.pareto)
+    def pareto(self, a, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.pareto, a,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.poisson)
+    def poisson(self, lam=1.0, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.poisson, lam,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.power)
+    def power(self, a, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.power, a,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.randint)
+    def randint(self, low, high=None, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.randint, low, high,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.random_integers)
+    def random_integers(self, low, high=None, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.random_integers, low, high,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.random_sample)
+    def random_sample(self, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.random_sample,
+                         size=size, chunks=chunks)
+
+    random = random_sample
+
+    @doc_wraps(np.random.RandomState.rayleigh)
+    def rayleigh(self, scale=1.0, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.rayleigh, scale,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.standard_cauchy)
+    def standard_cauchy(self, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.standard_cauchy,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.standard_exponential)
+    def standard_exponential(self, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.standard_exponential,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.standard_gamma)
+    def standard_gamma(self, shape, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.standard_gamma, shape,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.standard_normal)
+    def standard_normal(self, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.standard_normal,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.standard_t)
+    def standard_t(self, df, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.standard_t, df,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.tomaxint)
+    def tomaxint(self, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.tomaxint,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.triangular)
+    def triangular(self, left, mode, right, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.triangular, left, mode, right,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.uniform)
+    def uniform(self, low=0.0, high=1.0, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.uniform, low, high,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.vonmises)
+    def vonmises(self, mu, kappa, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.vonmises, mu, kappa,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.wald)
+    def wald(self, mean, scale, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.wald, mean, scale,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.weibull)
+    def weibull(self, a, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.weibull, a,
+                         size=size, chunks=chunks)
+
+    @doc_wraps(np.random.RandomState.zipf)
+    def zipf(self, a, size=None, chunks=None):
+        return self._wrap(np.random.RandomState.zipf, a,
+                         size=size, chunks=chunks)
+
+
+def _apply_random(func, seed, size, args, kwargs):
+    """ Apply RandomState method with seed
+
+    >>> _apply_random('normal', 123, 3, (10, 1.0), {})
+    array([  8.9143694 ,  10.99734545,  10.2829785 ])
+    """
+    state = np.random.RandomState(seed)
+    func = getattr(state, func)
+    return func(*args, size=size, **kwargs)
+
+
+_state = RandomState()
+
+
+beta = _state.beta
+binomial = _state.binomial
+chisquare = _state.chisquare
+exponential = _state.exponential
+f = _state.f
+gamma = _state.gamma
+geometric = _state.geometric
+gumbel = _state.gumbel
+hypergeometric = _state.hypergeometric
+laplace = _state.laplace
+logistic = _state.logistic
+lognormal = _state.lognormal
+logseries = _state.logseries
+negative_binomial = _state.negative_binomial
+noncentral_chisquare = _state.noncentral_chisquare
+noncentral_f = _state.noncentral_f
+normal = _state.normal
+pareto = _state.pareto
+poisson = _state.poisson
+power = _state.power
+rayleigh = _state.rayleigh
+random_sample = _state.random_sample
+random = random_sample
+triangular = _state.triangular
+uniform = _state.uniform
+vonmises = _state.vonmises
+wald = _state.wald
+weibull = _state.weibull
+zipf = _state.zipf
+
+"""
+Standard distributions
 """
+
+standard_cauchy = _state.standard_cauchy
+standard_exponential = _state.standard_exponential
+standard_gamma = _state.standard_gamma
+standard_normal = _state.standard_normal
+standard_t = _state.standard_t


Test patch:
diff --git a/dask/array/tests/test_linalg.py b/dask/array/tests/test_linalg.py
--- a/dask/array/tests/test_linalg.py
+++ b/dask/array/tests/test_linalg.py
@@ -85,7 +85,7 @@ def test_svd_compressed():
     mat = mat1.dot(mat2)
     data = from_array(mat, chunks=(50, 50))
 
-    n_iter = 4
+    n_iter = 6
     for i in range(n_iter):
         u, s, vt = svd_compressed(data, r)
         u = np.array(u)
@@ -97,7 +97,7 @@ def test_svd_compressed():
             usvt += np.dot(u, np.dot(np.diag(s), vt))
     usvt /= n_iter
 
-    tol = 1e-1
+    tol = 2e-1
     assert np.allclose(np.linalg.norm(mat - usvt),
                        np.linalg.norm(mat),
                        rtol=tol, atol=tol)  # average accuracy check
diff --git a/dask/array/tests/test_random.py b/dask/array/tests/test_random.py
--- a/dask/array/tests/test_random.py
+++ b/dask/array/tests/test_random.py
@@ -2,8 +2,52 @@
 pytest.importorskip('numpy')
 
 import numpy as np
+import dill
 from dask.array.core import Array
 from dask.array.random import random, exponential, normal
+import dask.array as da
+import dask
+from dask.multiprocessing import get as mpget
+
+
+def test_RandomState():
+    state = da.random.RandomState(5)
+    x = state.normal(10, 1, size=10, chunks=5)
+    assert (x.compute() == x.compute()).all()
+
+    state = da.random.RandomState(5)
+    y = state.normal(10, 1, size=10, chunks=5)
+    assert (x.compute() == y.compute()).all()
+
+
+def test_concurrency():
+    state = da.random.RandomState(5)
+    x = state.normal(10, 1, size=100, chunks=2)
+
+    state = da.random.RandomState(5)
+    y = state.normal(10, 1, size=100, chunks=2)
+    assert (x.compute(get=mpget) == y.compute(get=mpget)).all()
+
+
+def test_doc_randomstate():
+    assert 'mean' in da.random.RandomState(5).normal.__doc__
+
+
+def test_serializability():
+    state = da.random.RandomState(5)
+    x = state.normal(10, 1, size=10, chunks=5)
+
+    y = dill.loads(dill.dumps(x))
+
+    assert (x.compute() == y.compute()).all()
+
+
+def test_determinisim_through_dask_values():
+    samples_1 = da.random.RandomState(42).normal(size=1000, chunks=10)
+    samples_2 = da.random.RandomState(42).normal(size=1000, chunks=10)
+
+    assert [v for k, v in sorted(samples_1.dask.items())] ==\
+           [v for k, v in sorted(samples_2.dask.items())]
 
 
 def test_random():
@@ -39,16 +83,6 @@ def test_kwargs():
     assert 8 < x.mean() < 12
 
 
-def test_kwargs_size_or_shape():
-    a = normal(loc=10.0, scale=0.1, shape=(10, 10), chunks=(5, 5))
-    b = normal(loc=10.0, scale=0.1, size=(10, 10), chunks=(5, 5))
-    assert isinstance(a, Array)
-    assert isinstance(b, Array)
-    assert a.chunks == b.chunks
-
-    assert np.array(a).shape == np.array(b).shape
-
-
 def test_unique_names():
     a = random((10, 10), chunks=(5, 5))
     b = random((10, 10), chunks=(5, 5))