Repository: mars-project/mars

Issue:
[TEST]Specialized tests needed under Windows
As plasma store does not support Windows currently, cases for web sessions on a mock server are needed to make sure Mars client works properly under Windows.


Discussion:

Resolving patch:
diff --git a/mars/worker/service.py b/mars/worker/service.py
--- a/mars/worker/service.py
+++ b/mars/worker/service.py
@@ -18,7 +18,10 @@
 import logging
 import time
 
-from pyarrow import plasma
+try:
+    from pyarrow import plasma
+except ImportError:
+    plasma = None
 
 from ..config import options
 from .. import resource, kvstore
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -96,7 +96,7 @@ def execfile(fname, globs, locs=None):
 
 
 setup_options = dict(
-    name='mars',
+    name='pymars',
     version=version_ns['__version__'],
     description='MARS: a tensor-based unified framework for large-scale data computation.',
     long_description=long_description,


Test patch:
diff --git a/mars/actors/pool/tests/test_gevent_pool.py b/mars/actors/pool/tests/test_gevent_pool.py
--- a/mars/actors/pool/tests/test_gevent_pool.py
+++ b/mars/actors/pool/tests/test_gevent_pool.py
@@ -20,6 +20,7 @@
 import uuid
 import time
 import unittest
+import sys
 
 import gevent
 
@@ -172,6 +173,7 @@ def distribute(self, uid):
         return int(hashlib.sha1(to_binary(uid)).hexdigest(), 16) % (self.n_process - 1) + 1
 
 
+@unittest.skipIf(sys.platform == 'win32', 'does not run in windows')
 class Test(unittest.TestCase):
     def setUp(self):
         self.exceptions = gevent.hub.Hub.NOT_ERROR
diff --git a/mars/deploy/local/tests/test_cluster.py b/mars/deploy/local/tests/test_cluster.py
--- a/mars/deploy/local/tests/test_cluster.py
+++ b/mars/deploy/local/tests/test_cluster.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 import unittest
+import sys
 
 import numpy as np
 
@@ -43,6 +44,7 @@ def __init__(self, f=None, **kw):
         super(SerializeMustFailOperand, self).__init__(_f=f, **kw)
 
 
+@unittest.skipIf(sys.platform == 'win32', 'does not run in windows')
 class Test(unittest.TestCase):
     def setUp(self):
         self._old_cache_memory_limit = options.worker.cache_memory_limit
diff --git a/mars/scheduler/tests/test_assigner.py b/mars/scheduler/tests/test_assigner.py
--- a/mars/scheduler/tests/test_assigner.py
+++ b/mars/scheduler/tests/test_assigner.py
@@ -41,7 +41,7 @@ def get_reply(self):
 class Test(unittest.TestCase):
 
     def testAssignerActor(self):
-        with create_actor_pool(backend='gevent') as pool:
+        with create_actor_pool(n_process=1, backend='gevent') as pool:
             pool.create_actor(ClusterInfoActor, [pool.cluster_info.address],
                               uid=ClusterInfoActor.default_name())
             resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_name())
diff --git a/mars/scheduler/tests/test_main.py b/mars/scheduler/tests/test_main.py
--- a/mars/scheduler/tests/test_main.py
+++ b/mars/scheduler/tests/test_main.py
@@ -34,6 +34,7 @@
 from mars.scheduler.graph import GraphState
 
 
+@unittest.skipIf(sys.platform == 'win32', "plasma don't support windows")
 class Test(unittest.TestCase):
 
     @classmethod
diff --git a/mars/tensor/execution/tests/test_datasource_execute.py b/mars/tensor/execution/tests/test_datasource_execute.py
--- a/mars/tensor/execution/tests/test_datasource_execute.py
+++ b/mars/tensor/execution/tests/test_datasource_execute.py
@@ -54,7 +54,7 @@ def testCreateSparseExecution(self):
 
         res = self.executor.execute_tensor(t3)
         self.assertIsInstance(res[0], SparseNDArray)
-        self.assertEqual(res[0].dtype, np.int64)
+        self.assertEqual(res[0].dtype, np.int_)
         np.testing.assert_array_equal(res[0].toarray(), mat[..., :2].toarray())
         np.testing.assert_array_equal(res[1].toarray(), mat[..., 2:].toarray())
 
@@ -80,7 +80,7 @@ def testEmptyExecution(self):
 
         res = self.executor.execute_tensor(t, concat=True)
         self.assertEqual(res[0].shape, (20, 30))
-        self.assertTrue(res[0].dtype, np.int64)
+        self.assertEqual(res[0].dtype, np.int64)
         self.assertFalse(np.array_equal(res, np.zeros((20, 30))))
 
         t = empty((20, 30), chunks=5)
@@ -91,7 +91,7 @@ def testEmptyExecution(self):
         t2 = empty_like(t)
         res = self.executor.execute_tensor(t2, concat=True)
         self.assertEqual(res[0].shape, (20, 30))
-        self.assertTrue(res[0].dtype, np.int64)
+        self.assertEqual(res[0].dtype, np.float64)
 
     def testFullExecution(self):
         t = full((2, 2), 1, dtype='f4', chunks=1)
@@ -328,7 +328,7 @@ def testEyeExecution(self):
         t = eye(2, dtype=int)
 
         res = self.executor.execute_tensor(t, concat=True)[0]
-        self.assertEqual(res.dtype, np.int64)
+        self.assertEqual(res.dtype, np.int_)
 
         # test sparse
         t = eye(5, sparse=True, chunks=2)
@@ -410,7 +410,7 @@ def testLinspaceExecution(self):
         a = linspace(2.0, 9.0, num=11, chunks=3, dtype=int)
 
         res = self.executor.execute_tensor(a, concat=True)[0]
-        self.assertEqual(res.dtype, np.int64)
+        self.assertEqual(res.dtype, np.int_)
 
     def testMeshgridExecution(self):
         a = arange(5, chunks=2)
diff --git a/mars/tests/test_promise.py b/mars/tests/test_promise.py
--- a/mars/tests/test_promise.py
+++ b/mars/tests/test_promise.py
@@ -18,6 +18,7 @@
 import time
 import unittest
 import weakref
+import sys
 
 import gevent
 
@@ -87,6 +88,7 @@ def _raise_exception(exc):
     raise exc
 
 
+@unittest.skipIf(sys.platform == 'win32', 'does not run in windows')
 class Test(unittest.TestCase):
     def testPromise(self):
         promises = weakref.WeakValueDictionary()
diff --git a/mars/web/tests/test_api.py b/mars/web/tests/test_api.py
--- a/mars/web/tests/test_api.py
+++ b/mars/web/tests/test_api.py
@@ -14,25 +14,34 @@
 # limitations under the License.
 
 import time
+import requests
+import json
+import unittest
+import mock
 import os
 import sys
 import signal
 import subprocess
-import unittest
 
+import gevent
 import numpy as np
 from numpy.testing import assert_array_equal
-import gevent
-import requests
 
-from mars.config import options
+
 from mars import tensor as mt
+from mars.tensor.execution.core import Executor
+from mars.actors import create_actor_pool, new_client
 from mars.utils import get_next_port
-from mars.actors.core import new_client
-from mars.scheduler import KVStoreActor
+from mars.cluster_info import ClusterInfoActor
+from mars.scheduler import SessionManagerActor, KVStoreActor, ResourceActor
+from mars.scheduler.graph import GraphActor
+from mars.web import MarsWeb
 from mars.session import new_session
+from mars.serialize.dataserializer import dumps, loads
+from mars.config import options
 
 
+@unittest.skipIf(sys.platform == 'win32', 'does not run in windows')
 class Test(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -179,3 +188,67 @@ def testApi(self):
             res = requests.get(service_ep + '/worker')
             self.assertEqual(res.status_code, 200)
 
+
+class TestWithMockServer(unittest.TestCase):
+    def setUp(self):
+        self._executor = Executor('numpy')
+
+        # create scheduler pool with needed actor
+        scheduler_address = '127.0.0.1:' + str(get_next_port())
+        pool = create_actor_pool(address=scheduler_address, n_process=1, backend='gevent')
+        pool.create_actor(ClusterInfoActor, [scheduler_address], uid=ClusterInfoActor.default_name())
+        pool.create_actor(ResourceActor, uid=ResourceActor.default_name())
+        pool.create_actor(SessionManagerActor, uid=SessionManagerActor.default_name())
+        self._kv_store_ref = pool.create_actor(KVStoreActor, uid=KVStoreActor.default_name())
+        self._pool = pool
+
+        self.start_web(scheduler_address)
+
+        check_time = time.time()
+        while True:
+            if time.time() - check_time > 30:
+                raise SystemError('Wait for service start timeout')
+            try:
+                resp = requests.get(self._service_ep + '/api', timeout=1)
+            except (requests.ConnectionError, requests.Timeout):
+                time.sleep(1)
+                continue
+            if resp.status_code >= 400:
+                time.sleep(1)
+                continue
+            break
+
+    def tearDown(self):
+        self._web.stop()
+        self._pool.stop()
+
+    def start_web(self, scheduler_address):
+        import gevent.monkey
+        gevent.monkey.patch_all()
+
+        web_port = str(get_next_port())
+        mars_web = MarsWeb(port=int(web_port), scheduler_ip=scheduler_address)
+        mars_web.start()
+        service_ep = 'http://127.0.0.1:' + web_port
+        self._web = mars_web
+        self._service_ep = service_ep
+
+    @mock.patch(GraphActor.__module__ + '.GraphActor.execute_graph')
+    @mock.patch(GraphActor.__module__ + '.ResultReceiverActor.fetch_tensor')
+    def testApi(self, mock_fetch_tensor, _):
+        with new_session(self._service_ep) as sess:
+            self._kv_store_ref.write('/workers/meta/%s' % 'mock_endpoint', 'mock_meta')
+            self.assertEqual(sess.count_workers(), 1)
+
+            a = mt.ones((100, 100), chunks=30)
+            b = mt.ones((100, 100), chunks=30)
+            c = a.dot(b)
+            graph_key = sess.run(c, timeout=120, wait=False)
+            self._kv_store_ref.write('/sessions/%s/graph/%s/state' % (sess.session_id, graph_key), 'SUCCEEDED')
+            graph_url = '%s/api/session/%s/graph/%s' % (self._service_ep, sess.session_id, graph_key)
+            graph_state = json.loads(requests.get(graph_url).text)
+            self.assertEqual(graph_state['state'], 'success')
+            mock_fetch_tensor.return_value = dumps(self._executor.execute_tensor(c, concat=True)[0])
+            data_url = graph_url + '/data/' + c.key
+            data = loads(requests.get(data_url).content)
+            assert_array_equal(data, np.ones((100, 100)) * 100)