internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: mars-project/mars

Issue:
Implement __array_function__ protocol to provide interoperability with numpy
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Is your feature request related to a problem? Please describe.**

`__array_function__` is described in [NEP-18](https://www.numpy.org/neps/nep-0018-array-function-protocol.html). If our `Tensor` object implement such method, any function of numpy can be called on mars tensor, not only for the ufunc.

**Describe the solution you'd like**

Add `__array_function__` to `Tensor`.


Discussion:

Resolving patch:
diff --git a/mars/tensor/core.py b/mars/tensor/core.py
--- a/mars/tensor/core.py
+++ b/mars/tensor/core.py
@@ -248,6 +248,22 @@ def __array__(self, dtype=None):
         else:
             return np.asarray(self.execute(), dtype=dtype)
 
+    def __array_function__(self, func, types, args, kwargs):
+        from .. import tensor as module
+
+        for submodule in func.__module__.split('.')[1:]:
+            try:
+                module = getattr(module, submodule)
+            except AttributeError:
+                return NotImplemented
+        if not hasattr(module, func.__name__):
+            return NotImplemented
+        mars_func = getattr(module, func.__name__)
+        if mars_func is func:
+            # avoid Numpy func
+            return NotImplemented
+        return mars_func(*args, **kwargs)
+
     def transpose(self, *axes):
         """
         Returns a view of the tensor with axes transposed.


Test patch:
diff --git a/mars/tensor/execution/tests/test_core_execute.py b/mars/tensor/execution/tests/test_core_execute.py
--- a/mars/tensor/execution/tests/test_core_execute.py
+++ b/mars/tensor/execution/tests/test_core_execute.py
@@ -16,6 +16,8 @@
 
 import unittest
 
+import numpy as np
+
 from mars.executor import Executor
 from mars.tensor import ones
 from mars.session import LocalSession, Session
@@ -40,3 +42,13 @@ def testDecref(self):
         del b
         # decref called
         self.assertEqual(len(self.executor.chunk_result), 0)
+
+    def testArrayFunction(self):
+        a = ones((10, 20), chunk_size=5)
+
+        # test sum
+        self.assertEqual(np.sum(a).execute(), 200)
+
+        # test qr
+        q, r = np.linalg.qr(a)
+        self.assertTrue(np.allclose(np.dot(q, r), a).execute())
Repository: mars-project/mars

Issue:
[BUG] Random operands does not give size estimations
**Describe the bug**
Random generators (for instance, TensorRandInt or TensorRand) always provide 0 when calculating size estimations when ``sparse=True``. Close estimations can be worked out by multiplying sparsity with original size.


Discussion:

Resolving patch:
diff --git a/mars/tensor/execution/random.py b/mars/tensor/execution/random.py
--- a/mars/tensor/execution/random.py
+++ b/mars/tensor/execution/random.py
@@ -241,6 +241,14 @@ def _randint(ctx, chunk):
     return _rand(ctx, chunk)
 
 
+def _random_estimate_size(ctx, chunk):
+    if not chunk.is_sparse() or not getattr(chunk.op, '_density', None):
+        raise NotImplementedError
+    # use density to estimate real memory usage
+    nbytes = int(chunk.nbytes * getattr(chunk.op, '_density'))
+    ctx[chunk.key] = (nbytes, nbytes)
+
+
 def register_random_handler():
     from ...executor import register
     from ...operands import random as random_op
@@ -248,4 +256,4 @@ def register_random_handler():
     register(random_op.SimpleRandomData, _rand)
     register(random_op.Distribution, _distribution)
     register(random.TensorMultivariateNormal, _multivariate_normal)
-    register(random.TensorRandint, _randint)
+    register(random.TensorRandint, _randint, _random_estimate_size)


Test patch:
diff --git a/mars/tensor/execution/tests/test_random_execute.py b/mars/tensor/execution/tests/test_random_execute.py
--- a/mars/tensor/execution/tests/test_random_execute.py
+++ b/mars/tensor/execution/tests/test_random_execute.py
@@ -49,7 +49,12 @@ def testRandnExecution(self):
             self.assertTrue(np.array_equal(res, np.random.RandomState(0).randn(5, 5)))
 
     def testRandintExecution(self):
+        size_executor = Executor(sync_provider_type=Executor.SyncProviderType.MOCK)
+
         arr = tensor.random.randint(0, 2, size=(10, 30), chunk_size=3)
+        size_res = size_executor.execute_tensor(arr, mock=True)
+        self.assertEqual(arr.nbytes, sum(tp[0] for tp in size_res))
+
         res = self.executor.execute_tensor(arr, concat=True)[0]
         self.assertEqual(res.shape, (10, 30))
         self.assertTrue(np.all(res >= 0))
@@ -158,7 +163,12 @@ def testChoiceExecution(self):
                                                                     p=[.2, .5, .3])))
 
     def testSparseRandintExecution(self):
+        size_executor = Executor(sync_provider_type=Executor.SyncProviderType.MOCK)
+
         arr = tensor.random.randint(1, 2, size=(30, 50), density=.1, chunk_size=10, dtype='f4')
+        size_res = size_executor.execute_tensor(arr, mock=True)
+        self.assertAlmostEqual(arr.nbytes * 0.1, sum(tp[0] for tp in size_res))
+
         res = self.executor.execute_tensor(arr, concat=True)[0]
         self.assertTrue(issparse(res))
         self.assertEqual(res.shape, (30, 50))
Repository: mars-project/mars

Issue:
[BUG] hard_limit is not effective in worker
**Describe the bug**
As is implemented in

https://github.com/mars-project/mars/blob/a6077669bf50ebe99aaa286beadb3c2796640e1c/mars/worker/quota.py#L397

``mem_stats.total - self._overall_size`` is always larger than 0, as self._overall_size is equal to a proportion of mem_stats.total. Therefore available_size is always equal to mem_stats.available when nothing is held in MemQuotaActor. Thus hard_limit configuration for a worker takes no effect.


Discussion:
The line is not changed after merge, hence I reopen this issue.

Resolving patch:
diff --git a/mars/worker/quota.py b/mars/worker/quota.py
--- a/mars/worker/quota.py
+++ b/mars/worker/quota.py
@@ -391,7 +391,7 @@ def update_mem_stats(self):
     def _has_space(self, delta):
         mem_stats = resource.virtual_memory()
         # calc available physical memory
-        available_size = mem_stats.available - min(0, mem_stats.total - self._overall_size) \
+        available_size = mem_stats.available - max(0, mem_stats.total - self._overall_size) \
             - self._total_proc
         if max(delta, 0) >= available_size:
             logger.warning('%s met hard memory limitation: request %d, available %d, hard limit %d',
@@ -404,7 +404,7 @@ def _has_space(self, delta):
     def _log_allocate(self, msg, *args, **kwargs):
         mem_stats = resource.virtual_memory()
         # calc available physical memory
-        available_size = mem_stats.available - min(0, mem_stats.total - self._overall_size) \
+        available_size = mem_stats.available - max(0, mem_stats.total - self._overall_size) \
             - self._total_proc
         args += (self._allocated_size, self._total_size, mem_stats.available, available_size,
                  self._overall_size, self._total_proc)


Test patch:
Repository: mars-project/mars

Issue:
[BUG] compose=False is not effective in web session
**Describe the bug**
``compose=False`` is not effective in web session, as it passes the string ``False`` into server and the string is treated as True value.


Discussion:

Resolving patch:
diff --git a/mars/web/api.py b/mars/web/api.py
--- a/mars/web/api.py
+++ b/mars/web/api.py
@@ -93,7 +93,7 @@ def post(self, session_id):
         try:
             graph = self.get_argument('graph')
             target = self.get_argument('target').split(',')
-            compose = self.get_argument('compose', True)
+            compose = bool(int(self.get_argument('compose', '1')))
         except web.MissingArgumentError as ex:
             self.write(json.dumps(dict(msg=str(ex))))
             raise web.HTTPError(400, 'Argument missing')
diff --git a/mars/web/session.py b/mars/web/session.py
--- a/mars/web/session.py
+++ b/mars/web/session.py
@@ -221,7 +221,7 @@ def _submit_graph(self, graph_json, targets, compose=True):
         resp = self._req_session.post(session_url + '/graph', dict(
             graph=json.dumps(graph_json),
             target=targets,
-            compose=compose
+            compose='1' if compose else '0'
         ))
         if resp.status_code >= 400:
             resp_json = json.loads(resp.text)


Test patch:
Repository: mars-project/mars

Issue:
[BUG] UI colors are mismatch in task progress under python2.7
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
UI colors are mismatch in task progress.

![image](https://user-images.githubusercontent.com/14146443/60434463-c09b8f80-9c39-11e9-83f1-98523f60d5a9.png)





Discussion:

Resolving patch:
diff --git a/mars/scheduler/operands/core.py b/mars/scheduler/operands/core.py
--- a/mars/scheduler/operands/core.py
+++ b/mars/scheduler/operands/core.py
@@ -21,6 +21,7 @@
 
 
 class OperandState(Enum):
+    __order__ = 'UNSCHEDULED READY RUNNING FINISHED CACHED FREED FATAL CANCELLING CANCELLED'
     UNSCHEDULED = 'unscheduled'
     READY = 'ready'
     RUNNING = 'running'


Test patch:
Repository: mars-project/mars

Issue:
[BUG] Got wrong value when using negative indexing
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
Negative indexing got a wrong result, `slice_split` can't handle negative number properly.

**To Reproduce**
``` Python
In [1]: import numpy as np                                                                                                                           

In [2]: import mars.tensor as mt                                                                                                                     

In [3]: data = np.random.rand(10, 10)                                                                                                                

In [4]: t = mt.tensor(data, chunk_size=3)                                                                                                            

In [5]: t[-1].execute()                                                                                                                              
Out[5]: 
array([0.16859142, 0.02173202, 0.1544274 , 0.76698382, 0.37997933,
       0.44778154, 0.88055932, 0.30275961, 0.8652064 , 0.39251796])

In [6]: data[-1]                                                                                                                                     
Out[6]: 
array([0.79062543, 0.96115942, 0.7543363 , 0.25384024, 0.16946118,
       0.0266046 , 0.45766045, 0.35148908, 0.13565708, 0.59276878])

In [7]: data[2]                                                                                                                                      
Out[7]: 
array([0.16859142, 0.02173202, 0.1544274 , 0.76698382, 0.37997933,
       0.44778154, 0.88055932, 0.30275961, 0.8652064 , 0.39251796])

```



Discussion:

Resolving patch:
diff --git a/mars/tensor/expressions/utils.py b/mars/tensor/expressions/utils.py
--- a/mars/tensor/expressions/utils.py
+++ b/mars/tensor/expressions/utils.py
@@ -207,6 +207,7 @@ def slice_split(index, sizes):
     size = sum(sizes)
 
     if isinstance(index, Integral):
+        index = index if index >= 0 else size + index
         i = 0
         ind = index
         lens = list(sizes)


Test patch:
diff --git a/mars/tensor/execution/tests/test_index_execute.py b/mars/tensor/execution/tests/test_index_execute.py
--- a/mars/tensor/execution/tests/test_index_execute.py
+++ b/mars/tensor/execution/tests/test_index_execute.py
@@ -187,6 +187,10 @@ def testSliceExecution(self):
 
         np.testing.assert_array_equal(res[0], raw[2:9:2, 3:7, -1:-9:-2, 12:-11:-4])
 
+        arr3 = arr[-4, 2:]
+        res = self.executor.execute_tensor(arr3, concat=True)
+        np.testing.assert_equal(res[0], raw[-4, 2:])
+
         raw = sps.random(12, 14, density=.1)
         arr = tensor(raw, chunk_size=3)
Repository: mars-project/mars

Issue:
[BUG] Cannot import new_client
**Describe the bug**
When trying to import ``new_client`` from ``mars.actors`` in Python 2.7 in Linux, a ValueError is raised:

```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "mars/actors/core.pyx", line 147, in mars.actors.core.new_client
    cpdef object new_client(object parallel=None, str backend='gevent'):
  File "mars/actors/core.pyx", line 151, in mars.actors.core.new_client
    from .pool.gevent_pool import ActorClient
  File "mars/actors/pool/gevent_pool.pyx", line 38, in init mars.actors.pool.gevent_pool
    from ...lib import gipc
  File "mars/lib/gipc.pyx", line 1159, in init mars.lib.gipc
    __exec("""def _reraise(tp, value, tb=None):
  File "mars/lib/gipc.pyx", line 1150, in mars.lib.gipc.__exec
    frame = sys._getframe(1)
ValueError: call stack is not deep enough
```

**To Reproduce**
```python
>>> from mars.actors import new_client
>>> client = new_client()
```


Discussion:

Resolving patch:
diff --git a/mars/actors/__init__.py b/mars/actors/__init__.py
--- a/mars/actors/__init__.py
+++ b/mars/actors/__init__.py
@@ -14,8 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 from .core import create_actor_pool, Actor, FunctionActor, new_client, \
     register_actor_implementation, unregister_actor_implementation
 from .errors import ActorPoolNotStarted, ActorNotExist, ActorAlreadyExist
 from .distributor import Distributor
+
+# import gipc first to avoid stack issue of `call stack is not deep enough`
+try:
+    from ..lib import gipc
+    del gipc
+except ImportError:  # pragma: no cover
+    pass


Test patch:
Repository: mars-project/mars

Issue:
Running Mars under docker can use psutil.virtual_memory() directly
**Is your feature request related to a problem? Please describe.**
When running in Docker containers with ``MARS_USE_PROCESS_STAT`` on, both cpu and memory usage are calculated by summing results of all processes. This can be effective for cpu. However, this method may produce larger memory usages than actual.

**Describe the solution you'd like**
We can specify different methods to measure cpu and memory usage to obtain more accurate memory usage for most of the cases.


Discussion:

Resolving patch:
diff --git a/mars/resource.py b/mars/resource.py
--- a/mars/resource.py
+++ b/mars/resource.py
@@ -26,10 +26,12 @@
 _proc = psutil.Process()
 _timer = getattr(time, 'monotonic', time.time)
 
+_cpu_use_process_stat = bool(int(os.environ.get('MARS_CPU_USE_PROCESS_STAT', '0').strip('"')))
+_mem_use_process_stat = bool(int(os.environ.get('MARS_MEM_USE_PROCESS_STAT', '0').strip('"')))
+
 if 'MARS_USE_PROCESS_STAT' in os.environ:
-    _use_process_stat = bool(int(os.environ['MARS_USE_PROCESS_STAT'].strip('"')))
-else:
-    _use_process_stat = False
+    _cpu_use_process_stat = _mem_use_process_stat = \
+        bool(int(os.environ['MARS_USE_PROCESS_STAT'].strip('"')))
 
 if 'MARS_CPU_TOTAL' in os.environ:
     _cpu_total = int(os.environ['MARS_CPU_TOTAL'].strip('"'))
@@ -53,7 +55,7 @@
 
 def virtual_memory():
     sys_mem = psutil.virtual_memory()
-    if not _use_process_stat:
+    if not _mem_use_process_stat:
         total = sys_mem.total
         used = sys_mem.used + getattr(sys_mem, 'shared', 0)
         available = sys_mem.available
@@ -112,7 +114,7 @@ def timer():
 
 def cpu_percent():
     global _last_cpu_measure
-    if not _use_process_stat:
+    if not _cpu_use_process_stat:
         return sum(psutil.cpu_percent(percpu=True))
 
     num_cpus = cpu_count() or 1


Test patch:
Repository: mars-project/mars

Issue:
[BUG]The parameter `open_browser` of `new_cluster` doesn't work
**Describe the bug**

In `new_cluster`, we use 

```python
open_browser = open_browser or options.deploy.open_browser
```

to decide if we should open the browser after web worker available. When `open_browser` is `False`, it will still fall back to `options.deploy.open_browser` and open the browser.

**To Reproduce**

To help us reproducing this bug, please provide information below:
1. Your Python version: 3.7
2. The version of Mars you use: master
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

**Expected behavior**

The web browser shouldn't be opened when `open_browser` is `False`.


Discussion:

Resolving patch:
diff --git a/mars/deploy/local/core.py b/mars/deploy/local/core.py
--- a/mars/deploy/local/core.py
+++ b/mars/deploy/local/core.py
@@ -290,7 +290,8 @@ def stop(self):
 
 def new_cluster(address='0.0.0.0', web=False, n_process=None, shared_memory=None,
                 open_browser=None, **kw):
-    open_browser = open_browser or options.deploy.open_browser
+    if open_browser is None:
+        open_browser = options.deploy.open_browser
     endpoint = gen_endpoint(address)
     web_endpoint = None
     if web is True:


Test patch:
Repository: mars-project/mars

Issue:
[BUG] Worker's used memory is inaccurate
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
We use `mars.resource.virtual_memory()` to collect memory usage on each worker,  the value of `used` is larger than actual system used memory. Moreover, it seems that the numbers after the decimal point are always 0 in Python2.7. 

Web shows that:
![image](https://user-images.githubusercontent.com/14146443/61703485-3e077b00-ad74-11e9-9916-db260392414a.png)

psutil.virtual_memory:
```svmem(total=19327352832, available=13306626048, percent=31.2, used=10058821632, free=9268531200, active=673157120, inactive=9385664512, buffers=0, cached=14399549440, shared=9297842176, slab=6384566272)```


Discussion:

Resolving patch:
diff --git a/mars/utils.py b/mars/utils.py
--- a/mars/utils.py
+++ b/mars/utils.py
@@ -126,13 +126,13 @@ def readable_size(size):
     if size < 1024:
         return size
     elif 1024 <= size < 1024 ** 2:
-        return '{0:.2f}K'.format(size / 1024)
+        return '{0:.2f}K'.format(size * 1.0 / 1024)
     elif 1024 ** 2 <= size < 1024 ** 3:
-        return '{0:.2f}M'.format(size / (1024 ** 2))
+        return '{0:.2f}M'.format(size * 1.0 / (1024 ** 2))
     elif 1024 ** 3 <= size < 1024 ** 4:
-        return '{0:.2f}G'.format(size / (1024 ** 3))
+        return '{0:.2f}G'.format(size * 1.0 / (1024 ** 3))
     else:
-        return '{0:.2f}T'.format(size / (1024 ** 4))
+        return '{0:.2f}T'.format(size * 1.0 / (1024 ** 4))
 
 
 _commit_hash, _commit_ref = None, None


Test patch:
Repository: mars-project/mars

Issue:
We should use numpy.fft only because it is faster than scipy.fftpack
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

Maybe at a point in the history, scipy's fftpack is faster than numpy's fft module. Thus we do some optimization in the execution of fft that  if scipy is installed, scipy's fftpack would be used to calculate fft. However, recently, I found that numpy's fft is obviously faster than scipy's fftpack. Sample code is shown below.

```python
In [1]: N = 1600000                                                             

In [2]: import numpy as np                                                      

In [3]: a = np.random.rand(N, 10)                                               

In [4]: %timeit np.fft.fft(a)                                                   
118 ms ± 1.33 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)

In [6]: import scipy.fftpack as sfft                                            

In [7]: %timeit sfft.fft(a)                                                     
290 ms ± 3.65 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [9]: np.testing.assert_allclose(np.fft.fft(a), sfft.fft(a))
```

Hence, I suggest to use numpy only to calculate fft.



Discussion:

Resolving patch:
diff --git a/mars/tensor/fft/core.py b/mars/tensor/fft/core.py
--- a/mars/tensor/fft/core.py
+++ b/mars/tensor/fft/core.py
@@ -21,14 +21,8 @@
     Int64Field, ListField
 from ..utils import validate_axis, decide_chunk_sizes, recursive_tile
 from ..operands import TensorHasInput, TensorOperandMixin
-import numpy as np
 from ..array_utils import get_array_module
 
-try:
-    import scipy.fftpack as scifft
-except ImportError:  # pragma: no cover
-    scifft = None
-
 
 class TensorFFTBaseMixin(TensorOperandMixin):
     __slots__ = ()
@@ -246,23 +240,8 @@ def execute(cls, ctx, op):
 
 
 def _get_fft_func(op, xp):
-    from .. import fft as fftop
-
     fun_name = type(op).__name__.lower()[6:]  # all op starts with tensor
-    if type(op) in (fftop.TensorFFT, fftop.TensorIFFT, fftop.TensorFFT2, fftop.TensorIFFT2,
-                    fftop.TensorFFTN, fftop.TensorIFFTN):
-        if xp is np and scifft and op.norm is None:
-            def f(*args, **kwargs):
-                kwargs.pop('norm', None)
-                if 's' in kwargs:
-                    kwargs['shape'] = kwargs.pop('s', None)
-                return getattr(scifft, fun_name)(*args, **kwargs)
-
-            return f
-        else:
-            return getattr(xp.fft, fun_name)
-    else:
-        return getattr(xp.fft, fun_name)
+    return getattr(xp.fft, fun_name)
 
 
 class TensorStandardFFT(TensorBaseSingleDimensionFFT):


Test patch: