Repository: dask/distributed

Issue:
CPU affinity of multiple workers on one node
I'm experiencing a strange behaviour with (broadcast) scatter.
For development, I launch four workers on my PC (so they share one IP address).
The PC has 2 CPUs, each with 2 cores (so 4 hyper-threads in total).
Because the work for each worker cannot benefit from multithreading, I assign just one core to each worker (`ncores=1`).
If I distribute 4 jobs for them using a simple submit, each worker gets one job - that's fine.
However if I use scatter with broadcast and then use the data future from the scatter, the CPU affinity gets weird, just 2 cores are executing the code at once and the whole process is two times longer, so the workers probably aren't all working in parallel.
I'm using the broadcast in this way because of the use case that I described in issue #85.

Here's a toy MWE:

``` python
import distributed
from tornado.ioloop import IOLoop
from threading import Thread

loop = IOLoop()
t = Thread(target=loop.start)
t.daemon = True
t.start()

scheduler = distributed.Scheduler(ip="127.0.0.1")
scheduler.start(1800)

workers = [distributed.Nanny("127.0.0.1", 1800, ncores=1) for i in xrange(4)]
map(lambda w: w.start(0), workers)


class Test(object):
    def __init__(self, x):
        self.x = x


def fn(test, y):
    result = 1
    for x in xrange(10000):
        for u in xrange(50):
            result *= test.x + y * u
    return result

executor = distributed.Executor("127.0.0.1:1800")
test = Test(10)

# Variant A
#[data_future] = executor.scatter([test], broadcast=True)
#tasks = [executor.submit(fn, data_future, i) for i in xrange(4)]

# Variant B
tasks = [executor.submit(fn, test, i) for i in xrange(4)]

data = executor.gather(tasks)
```

Variant A takes two times longer and doesn't execute on all of the cores at once.
I know that this isn't probably a typical usage for distributed, but I don't see why it would behave in this way.
Am I missing something?



Discussion:

Resolving patch:
diff --git a/distributed/protocol.py b/distributed/protocol.py
--- a/distributed/protocol.py
+++ b/distributed/protocol.py
@@ -45,6 +45,7 @@
 
 default_compression = None
 
+
 with ignoring(ImportError):
     import zlib
     compressions['zlib'] = {'compress': zlib.compress,
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3,16 +3,17 @@
 from collections import defaultdict, deque
 from datetime import datetime, timedelta
 import logging
+import math
 import random
 import socket
 from time import time
 from timeit import default_timer
 
 try:
-    from cytoolz import frequencies
+    from cytoolz import frequencies, topk
 except ImportError:
-    from toolz import frequencies
-from toolz import memoize, valmap, first, second, keymap
+    from toolz import frequencies, topk
+from toolz import memoize, valmap, first, second, keymap, unique
 from tornado import gen
 from tornado.gen import Return
 from tornado.queues import Queue
@@ -28,7 +29,7 @@
 from .core import (rpc, connect, read, write, MAX_BUFFER_SIZE,
         Server, send_recv, coerce_to_address, error_message)
 from .utils import (All, ignoring, clear_queue, get_ip, ignore_exceptions,
-        ensure_ip, log_errors, key_split)
+        ensure_ip, log_errors, key_split, mean, divide_n_among_bins)
 
 
 logger = logging.getLogger(__name__)
@@ -370,8 +371,9 @@ def mark_ready_to_run(self, key):
 
         if self.dependencies.get(key, None) or key in self.restrictions:
             new_worker = decide_worker(self.dependencies, self.stacks,
-                    self.who_has, self.restrictions, self.loose_restrictions,
-                    self.nbytes, key)
+                    self.processing, self.who_has, self.has_what,
+                    self.restrictions, self.loose_restrictions, self.nbytes,
+                    key)
             if not new_worker:
                 self.unrunnable.add(key)
             else:
@@ -380,7 +382,7 @@ def mark_ready_to_run(self, key):
                 # self.ensure_occupied(new_worker)
         else:
             self.ready.appendleft(key)
-            self.ensure_idle_ready()
+            # self.ensure_idle_ready()
 
     def ensure_idle_ready(self):
         """ Run ready tasks on idle workers
@@ -400,9 +402,38 @@ def ensure_idle_ready(self):
         certain workers.
         """
         for worker in self.maybe_ready:
-            self.ensure_occupied(worker)
+            self.ensure_occupied_stacks(worker)
         self.maybe_ready.clear()
 
+        if self.idle and self.ready:
+            if len(self.ready) < len(self.idle):
+                def keyfunc(w):
+                    return (-len(self.stacks[w]) - len(self.processing[w]),
+                            -len(self.has_what.get(w, ())))
+                for worker in topk(len(self.ready), self.idle, key=keyfunc):
+                    self.ensure_occupied_ready_count(worker, count=1)
+            else:
+                workers = list(self.idle)
+                tpc = math.ceil(mean(map(self.tasks_per_core, workers))) # tasks per core
+                free_slots = [tpc * self.ncores[w] - len(self.processing[w])
+                              for w in workers]
+
+                # Clean out workers that *are* actually full
+                workers2 = []
+                free_slots2 = []
+                for w, fs in zip(workers, free_slots):
+                    if fs > 0:
+                        workers2.append(w)
+                        free_slots2.append(fs)
+                    else:
+                        self.idle.remove(w)
+
+                if workers2:
+                    n = min(sum(free_slots2), len(self.ready))
+                    counts = divide_n_among_bins(n, free_slots2)
+                    for worker, count in zip(workers2, counts):
+                        self.ensure_occupied_ready_count(worker, count=count)
+
         if 0 < len(self.idle) < len(self.ncores) and not self.ready:
             n = sum(map(len, self.stacks)) * len(self.idle) / len(self.ncores)
 
@@ -428,10 +459,6 @@ def ensure_idle_ready(self):
                 if n <= 0:
                     break
 
-        while self.idle and self.ready and self.ncores:
-            worker = min(self.idle, key=lambda w: len(self.has_what[w]))
-            self.ensure_occupied(worker)
-
     def mark_key_in_memory(self, key, workers=None, type=None):
         """ Mark that a key now lives in distributed memory """
         logger.debug("Mark %s in memory", key)
@@ -488,7 +515,7 @@ def release_live_dependencies(self, key):
                 if not s and dep and dep not in self.who_wants:
                     self.delete_data(keys=[dep])
 
-    def task_load(self, address):
+    def tasks_per_core(self, address):
         if time() - self.worker_info[address].get('last-task', 0) > 2:
             tasks_per_core = 2
         else:
@@ -501,9 +528,13 @@ def task_load(self, address):
             latency = max(latency, 10e-3)
             tasks_per_core = max(2, latency // duration)
             tasks_per_core = min(500, tasks_per_core)
-        return tasks_per_core * self.ncores[address]
+        return tasks_per_core
 
     def ensure_occupied(self, worker):
+        self.ensure_occupied_stacks(worker)
+        self.ensure_occupied_ready(worker)
+
+    def ensure_occupied_stacks(self, worker):
         """ Send tasks to worker while it has tasks and free cores
 
         These tasks may come from the worker's own stacks or from the global
@@ -511,13 +542,32 @@ def ensure_occupied(self, worker):
 
         We update the idle workers set appropriately.
         """
-        logger.debug('Ensure worker is occupied: %s', worker)
+        load = self.tasks_per_core(worker) * self.ncores[worker]
+        stack = self.stacks[worker]
 
-        load = self.task_load(worker)
-
-        while (self.stacks[worker] and
+        while (stack and
                load * self.ncores[worker] > len(self.processing[worker])):
-            key = self.stacks[worker].pop()
+            key = stack.pop()
+            if key not in self.tasks:
+                continue
+            if self.who_has.get(key):
+                continue
+            self.processing[worker].add(key)
+            logger.debug("Send job to worker: %s, %s", worker, key)
+            try:
+                self.send_task_to_worker(worker, key)
+            except StreamClosedError:
+                self.remove_worker(worker)
+                return
+
+        self._check_idle(worker, load)
+
+    def ensure_occupied_ready_count(self, worker, count):
+        for i in range(count):
+            try:
+                key = self.ready.pop()
+            except KeyError:
+                break
             if key not in self.tasks:
                 continue
             if self.who_has.get(key):
@@ -530,6 +580,11 @@ def ensure_occupied(self, worker):
                 self.remove_worker(worker)
                 return
 
+        self._check_idle(worker)
+
+    def ensure_occupied_ready(self, worker):
+        load = self.tasks_per_core(worker) * self.ncores[worker]
+
         while (self.ready and
                load * self.ncores[worker] > len(self.processing[worker])):
             key = self.ready.pop()
@@ -545,6 +600,11 @@ def ensure_occupied(self, worker):
                 self.remove_worker(worker)
                 return
 
+        self._check_idle(worker, load)
+
+    def _check_idle(self, worker, load=None):
+        if load is None:
+            load = self.tasks_per_core(worker) * self.ncores[worker]
         if load * self.ncores[worker] > len(self.processing[worker]):
             self.idle.add(worker)
         elif worker in self.idle:
@@ -893,6 +953,7 @@ def update_graph(self, client=None, tasks=None, keys=None,
             if k in self.tasks:
                 del tasks[k]
 
+        original_keys = keys
         keys = set(keys)
         for k in keys:
             self.who_wants[k].add(client)
@@ -946,8 +1007,9 @@ def update_graph(self, client=None, tasks=None, keys=None,
             if loose_restrictions:
                 self.loose_restrictions |= set(loose_restrictions)
 
-        for key in keys:
-            self.ensure_in_play(key)
+        for key in unique(original_keys):
+            if key in keys:
+                self.ensure_in_play(key)
 
         for key in touched | keys:
             for dep in self.dependencies[key]:
@@ -1048,6 +1110,9 @@ def forget(self, key):
         if key in self.who_has:
             self.delete_data(keys=[key])
 
+        if key in self.nbytes:
+            del self.nbytes[key]
+
         for plugin in self.plugins:
             try:
                 plugin.forget(self, key)
@@ -1069,7 +1134,7 @@ def cancel_key(self, key, client, retries=5):
 
     def cancel(self, stream, keys=None, client=None):
         """ Stop execution on a list of keys """
-        logger.info("Client %s requests to cancel keys %s", client, keys)
+        logger.info("Client %s requests to cancel %d keys", client, len(keys))
         for key in keys:
             self.cancel_key(key, client)
 
@@ -1162,7 +1227,7 @@ def handle_messages(self, in_queue, report, client=None):
             while True:
                 try:
                     msgs = yield next_message()  # in_queue.get()
-                except (StreamClosedError, AssertionError):
+                except (StreamClosedError, AssertionError, GeneratorExit):
                     break
                 except Exception as e:
                     logger.exception(e)
@@ -1709,35 +1774,38 @@ def replicate(self, stream=None, keys=None, n=None, workers=None, branching_fact
                         self.add_keys(address=w, keys=list(gathers[w]))
 
 
-def decide_worker(dependencies, stacks, who_has, restrictions,
+def decide_worker(dependencies, stacks, processing, who_has, has_what, restrictions,
                   loose_restrictions, nbytes, key):
     """ Decide which worker should take task
 
     >>> dependencies = {'c': {'b'}, 'b': {'a'}}
     >>> stacks = {'alice:8000': ['z'], 'bob:8000': []}
+    >>> processing = {'alice:8000': set(), 'bob:8000': set()}
     >>> who_has = {'a': {'alice:8000'}}
+    >>> has_what = {'alice:8000': {'a'}}
     >>> nbytes = {'a': 100}
     >>> restrictions = {}
     >>> loose_restrictions = set()
 
     We choose the worker that has the data on which 'b' depends (alice has 'a')
 
-    >>> decide_worker(dependencies, stacks, who_has, restrictions,
-    ...               loose_restrictions, nbytes, 'b')
+    >>> decide_worker(dependencies, stacks, processing, who_has, has_what,
+    ...               restrictions, loose_restrictions, nbytes, 'b')
     'alice:8000'
 
     If both Alice and Bob have dependencies then we choose the less-busy worker
 
     >>> who_has = {'a': {'alice:8000', 'bob:8000'}}
-    >>> decide_worker(dependencies, stacks, who_has, restrictions,
-    ...               loose_restrictions, nbytes, 'b')
+    >>> has_what = {'alice:8000': {'a'}, 'bob:8000': {'a'}}
+    >>> decide_worker(dependencies, stacks, processing, who_has, has_what,
+    ...               restrictions, loose_restrictions, nbytes, 'b')
     'bob:8000'
 
     Optionally provide restrictions of where jobs are allowed to occur
 
     >>> restrictions = {'b': {'alice', 'charlie'}}
-    >>> decide_worker(dependencies, stacks, who_has, restrictions,
-    ...               loose_restrictions, nbytes, 'b')
+    >>> decide_worker(dependencies, stacks, processing, who_has, has_what,
+    ...               restrictions, loose_restrictions, nbytes, 'b')
     'alice:8000'
 
     If the task requires data communication, then we choose to minimize the
@@ -1746,10 +1814,12 @@ def decide_worker(dependencies, stacks, who_has, restrictions,
 
     >>> dependencies = {'c': {'a', 'b'}}
     >>> who_has = {'a': {'alice:8000'}, 'b': {'bob:8000'}}
+    >>> has_what = {'alice:8000': {'a'}, 'bob:8000': {'b'}}
     >>> nbytes = {'a': 1, 'b': 1000}
     >>> stacks = {'alice:8000': [], 'bob:8000': []}
 
-    >>> decide_worker(dependencies, stacks, who_has, {}, set(), nbytes, 'c')
+    >>> decide_worker(dependencies, stacks, processing, who_has, has_what,
+    ...               {}, set(), nbytes, 'c')
     'bob:8000'
     """
     deps = dependencies[key]
@@ -1764,8 +1834,8 @@ def decide_worker(dependencies, stacks, who_has, restrictions,
             workers = {w for w in stacks if w in r or w.split(':')[0] in r}
             if not workers:
                 if key in loose_restrictions:
-                    return decide_worker(dependencies, stacks, who_has,
-                                         {}, set(), nbytes, key)
+                    return decide_worker(dependencies, stacks, processing,
+                            who_has, has_what, {}, set(), nbytes, key)
                 else:
                     return None
     if not workers or not stacks:
@@ -1781,7 +1851,10 @@ def decide_worker(dependencies, stacks, who_has, restrictions,
     minbytes = min(commbytes.values())
 
     workers = {w for w, nb in commbytes.items() if nb == minbytes}
-    worker = min(workers, key=lambda w: len(stacks[w]))
+    def objective(w):
+        return (len(stacks[w]) + len(processing[w]),
+                len(has_what.get(w, ())))
+    worker = min(workers, key=objective)
     return worker
 
 
diff --git a/distributed/utils.py b/distributed/utils.py
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -470,3 +470,30 @@ def ensure_bytes(s):
         return s.encode()
     raise TypeError(
             "Object %s is neither a bytes object nor has an encode method" % s)
+
+
+def divide_n_among_bins(n, bins):
+    """
+
+    >>> divide_n_among_bins(12, [1, 1])
+    [6, 6]
+    >>> divide_n_among_bins(12, [1, 2])
+    [4, 8]
+    >>> divide_n_among_bins(12, [1, 2, 1])
+    [3, 6, 3]
+    >>> divide_n_among_bins(11, [1, 2, 1])
+    [2, 6, 3]
+    """
+    total = sum(bins)
+    acc = 0.0
+    out = []
+    for b in bins:
+        now = n / total * b + acc
+        now, acc = divmod(now, 1)
+        out.append(int(now))
+    return out
+
+
+def mean(seq):
+    seq = list(seq)
+    return sum(seq) / len(seq)


Test patch:
diff --git a/distributed/tests/test_executor.py b/distributed/tests/test_executor.py
--- a/distributed/tests/test_executor.py
+++ b/distributed/tests/test_executor.py
@@ -1580,7 +1580,7 @@ def test_forget_simple(e, s, a, b):
     for coll in [s.tasks, s.dependencies, s.dependents, s.waiting,
             s.waiting_data, s.who_has, s.restrictions, s.loose_restrictions,
             s.released, s.keyorder, s.exceptions, s.who_wants,
-            s.exceptions_blame]:
+            s.exceptions_blame, s.nbytes]:
         assert x.key not in coll
         assert z.key not in coll
 
@@ -2015,7 +2015,7 @@ def g():
         raise StopIteration(3)  # py2.7 compat.
     f1 = e.map(lambda x: x, g())
     assert isinstance(f1, Iterator)
-    
+
     start = time()  # ensure that we compute eagerly
     while not s.tasks:
         yield gen.sleep(0.01)
@@ -2715,20 +2715,20 @@ def test_executor_replicate_sync(loop):
 
 
 @gen_cluster(executor=True, ncores=[('127.0.0.1', 4)] * 1)
-def test_task_load(e, s, a):
-    assert 4 < s.task_load(a.address) < 20
+def test_tasks_per_core(e, s, a):
+    assert 1 < s.tasks_per_core(a.address) < 4
     L = e.map(inc, range(100))  # very fast
     yield _wait(L)
     assert 0 < s.worker_info[a.address]['avg-task-duration'] < 0.1
 
-    assert 8 <= s.task_load(a.address) < 10000
+    assert 2 <= s.tasks_per_core(a.address) < 10000
 
     L = e.map(sleep, [0.1] * 10, pure=False)
     yield _wait(L)
 
     assert 0.0001 < s.worker_info[a.address]['avg-task-duration'] < 0.2
 
-    assert 4 < s.task_load(a.address) < 100
+    assert 1 < s.tasks_per_core(a.address) < 25
 
 
 @gen_cluster(executor=True, ncores=[('127.0.0.1', 4)] * 1)
@@ -2748,3 +2748,62 @@ def test_task_load_adapts_quickly_after_delay(e, s, a):
     future = e.submit(inc, 2)  # fast but soon, defers to old average
     yield _wait(future)
     assert 0.2 < s.worker_info[a.address]['avg-task-duration'] < 1
+
+
+@gen_cluster(executor=True, ncores=[('127.0.0.1', 4)] * 1)
+def test_task_load_adapts_quickly(e, s, a):
+    future = e.submit(slowinc, 1, delay=0.2)  # slow
+    yield _wait(future)
+    assert 0.15 < s.worker_info[a.address]['avg-task-duration'] < 0.4
+
+    futures = e.map(inc, range(10))  # very fast
+    yield _wait(futures)
+
+    assert 0 < s.worker_info[a.address]['avg-task-duration'] < 0.1
+
+
+@gen_cluster(executor=True, ncores=[('127.0.0.1', 1)] * 2)
+def test_even_load_after_fast_functions(e, s, a, b):
+    x = e.submit(inc, 1, workers=a.address)  # very fast
+    y = e.submit(inc, 2, workers=b.address)  # very fast
+    yield _wait([x, y])
+    assert 0 < s.worker_info[a.address]['avg-task-duration'] < 0.1
+    assert 0 < s.worker_info[b.address]['avg-task-duration'] < 0.1
+
+    futures = e.map(inc, range(2, 11))
+    yield _wait(futures)
+    assert len(a.data) == len(b.data)
+
+
+@gen_cluster(executor=True, ncores=[('127.0.0.1', 1)] * 2)
+def test_even_load_on_startup(e, s, a, b):
+    x, y = e.map(inc, [1, 2])
+    yield _wait([x, y])
+    assert len(a.data) == len(b.data) == 1
+
+
+@gen_cluster(executor=True, ncores=[('127.0.0.1', 2)] * 2)
+def test_contiguous_load(e, s, a, b):
+    w, x, y, z = e.map(inc, [1, 2, 3, 4])
+    yield _wait([w, x, y, z])
+
+    groups = [set(a.data), set(b.data)]
+    assert {w.key, x.key} in groups
+    assert {y.key, z.key} in groups
+
+
+@gen_cluster(executor=True, ncores=[('127.0.0.1', 1)] * 4)
+def test_balanced_with_submit(e, s, *workers):
+    L = [e.submit(slowinc, i) for i in range(4)]
+    yield _wait(L)
+    for w in workers:
+        assert len(w.data) == 1
+
+
+@gen_cluster(executor=True, ncores=[('127.0.0.1', 1)] * 4)
+def test_balanced_with_submit_and_resident_data(e, s, *workers):
+    [x] = yield e._scatter([10], broadcast=True)
+    L = [e.submit(slowinc, x, pure=False) for i in range(4)]
+    yield _wait(L)
+    for w in workers:
+        assert len(w.data) == 2
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -202,12 +202,14 @@ def test_decide_worker_with_many_independent_leaves():
                 {('x', i): i for i in range(100)})
     dependencies, dependents = get_deps(dsk)
     stacks = {alice: [], bob: []}
+    processing = {alice: set(), bob: set()}
     who_has = merge({('x', i * 2): {alice} for i in range(50)},
                     {('x', i * 2 + 1): {bob} for i in range(50)})
     nbytes = {k: 0 for k in who_has}
 
     for key in dsk:
-        worker = decide_worker(dependencies, stacks, who_has, {}, set(), nbytes, key)
+        worker = decide_worker(dependencies, stacks, processing, who_has, {},
+                               {}, set(), nbytes, key)
         stacks[worker].append(key)
 
     nhits = (len([k for k in stacks[alice] if alice in who_has[('x', k[1])]])
@@ -220,21 +222,25 @@ def test_decide_worker_with_restrictions():
     dependencies = {'x': set()}
     alice, bob, charlie = 'alice:8000', 'bob:8000', 'charlie:8000'
     stacks = {alice: [], bob: [], charlie: []}
+    processing = {alice: set(), bob: set(), charlie: set()}
     who_has = {}
     restrictions = {'x': {'alice', 'charlie'}}
     nbytes = {}
-    result = decide_worker(dependencies, stacks, who_has, restrictions, set(), nbytes, 'x')
+    result = decide_worker(dependencies, stacks, processing, who_has, {},
+                           restrictions, set(), nbytes, 'x')
     assert result in {alice, charlie}
 
 
     stacks = {alice: [1, 2, 3], bob: [], charlie: [4, 5, 6]}
-    result = decide_worker(dependencies, stacks, who_has, restrictions, set(), nbytes, 'x')
+    result = decide_worker(dependencies, stacks, processing, who_has, {},
+                           restrictions, set(), nbytes, 'x')
     assert result in {alice, charlie}
 
     dependencies = {'x': {'y'}}
     who_has = {'y': {bob}}
     nbytes = {'y': 0}
-    result = decide_worker(dependencies, stacks, who_has, restrictions, set(), nbytes, 'x')
+    result = decide_worker(dependencies, stacks, processing, who_has, {},
+                           restrictions, set(), nbytes, 'x')
     assert result in {alice, charlie}
 
 
@@ -242,32 +248,33 @@ def test_decide_worker_with_loose_restrictions():
     dependencies = {'x': set()}
     alice, bob, charlie = 'alice:8000', 'bob:8000', 'charlie:8000'
     stacks = {alice: [1, 2, 3], bob: [], charlie: [1]}
+    processing = {alice: set(), bob: set(), charlie: set()}
     who_has = {}
     nbytes = {}
     restrictions = {'x': {'alice', 'charlie'}}
 
-    result = decide_worker(dependencies, stacks, who_has, restrictions,
-                           set(), nbytes, 'x')
+    result = decide_worker(dependencies, stacks, processing, who_has, {},
+                           restrictions, set(), nbytes, 'x')
     assert result == charlie
 
-    result = decide_worker(dependencies, stacks, who_has, restrictions,
-                           {'x'}, nbytes, 'x')
+    result = decide_worker(dependencies, stacks, processing, who_has, {},
+                           restrictions, {'x'}, nbytes, 'x')
     assert result == charlie
 
     restrictions = {'x': {'david', 'ethel'}}
-    result = decide_worker(dependencies, stacks, who_has, restrictions,
-                           set(), nbytes, 'x')
+    result = decide_worker(dependencies, stacks, processing, who_has, {},
+                           restrictions, set(), nbytes, 'x')
     assert result is None
 
     restrictions = {'x': {'david', 'ethel'}}
-    result = decide_worker(dependencies, stacks, who_has, restrictions,
-                           {'x'}, nbytes, 'x')
+    result = decide_worker(dependencies, stacks, processing, who_has, {},
+                           restrictions, {'x'}, nbytes, 'x')
     assert result == bob
 
 
 
 def test_decide_worker_without_stacks():
-    assert not decide_worker({'x': []}, [], {}, {}, set(), {}, 'x')
+    assert not decide_worker({'x': []}, {}, {}, {}, {}, {}, set(), {}, 'x')
 
 
 def test_validate_state():