internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: mars-project/mars

Issue:
Remove conjugate in Cholesky decomposition which effects performance heavily
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

In the tile of Cholesky decomposition, we do a conjugate after transpose which is unnecessary and effect performance heavily.



Discussion:

Resolving patch:
diff --git a/mars/tensor/linalg/cholesky.py b/mars/tensor/linalg/cholesky.py
--- a/mars/tensor/linalg/cholesky.py
+++ b/mars/tensor/linalg/cholesky.py
@@ -25,16 +25,6 @@
 from ..array_utils import as_same_device, device
 
 
-def _H(chunk):
-    from ..base.transpose import TensorTranspose
-    from ..arithmetic.conj import TensorConj
-
-    trans_op = TensorTranspose(dtype=chunk.dtype)
-    c = trans_op.new_chunk([chunk], shape=chunk.shape[::-1], index=chunk.index[::-1])
-    conj_op = TensorConj(dtype=c.dtype)
-    return conj_op.new_chunk([c], shape=c.shape, index=c.index)
-
-
 class TensorCholesky(TensorHasInput, TensorOperandMixin):
     _op_type_ = OperandDef.CHOLESKY
 
@@ -60,6 +50,8 @@ def tile(cls, op):
         from ..datasource.zeros import TensorZeros
         from ..arithmetic.subtract import TensorSubtract
         from ..arithmetic.utils import tree_add
+        from ..base import TensorTranspose
+        from ..utils import reverse_order
         from .dot import TensorDot
         from .solve_triangular import TensorSolveTriangular
 
@@ -100,7 +92,9 @@ def tile(cls, op):
                             [target, s], shape=target.shape, order=tensor.order)
                     lower_chunk = TensorCholesky(lower=True, dtype=tensor.dtype).new_chunk(
                         [target], shape=target.shape, index=(i, j), order=tensor.order)
-                    upper_chunk = _H(lower_chunk)
+                    upper_chunk = TensorTranspose(dtype=lower_chunk.dtype).new_chunk(
+                        [lower_chunk], shape=lower_chunk.shape[::-1],
+                        index=lower_chunk.index[::-1], order=reverse_order(lower_chunk.order))
                     lower_chunks[lower_chunk.index] = lower_chunk
                     upper_chunks[upper_chunk.index] = upper_chunk
                 else:
@@ -122,7 +116,9 @@ def tile(cls, op):
                     upper_chunk = TensorSolveTriangular(lower=True, dtype=tensor.dtype).new_chunk(
                         [lower_chunks[j, j], target], shape=target.shape,
                         index=(j, i), order=tensor.order)
-                    lower_chunk = _H(upper_chunk)
+                    lower_chunk = TensorTranspose(dtype=upper_chunk.dtype).new_chunk(
+                        [upper_chunk], shape=upper_chunk.shape[::-1],
+                        index=upper_chunk.index[::-1], order=reverse_order(upper_chunk.order))
                     lower_chunks[lower_chunk.index] = lower_chunk
                     upper_chunks[upper_chunk.index] = upper_chunk
 


Test patch:
Repository: mars-project/mars

Issue:
[BUG] Set value to copied tensor still changed the original tensor
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

Set value to copied tensor still changed the original tensor.

**To Reproduce**


```
In [55]: a = mt.tensor([[1, 2], [3, 4]])                                        

In [56]: b = a[:2, :1]                                                          

In [57]: c = b.copy()                                                           

In [58]: c[0, 0] = 5                                                            

In [59]: a.execute()                                                            
Out[59]: 
array([[5, 2],  # wrong, should be [1, 2]
       [3, 4]])
```

**Expected behavior**

Copied tensor should not share the same data with original one.




Discussion:

Resolving patch:
diff --git a/mars/core.py b/mars/core.py
--- a/mars/core.py
+++ b/mars/core.py
@@ -604,6 +604,9 @@ def view(self):
 
     def copy(self):
         new_op = self.op.copy().reset_key()
+        if new_op.create_view:
+            # set create_view to False
+            new_op._create_view = False
         new_outs = new_op.new_tileables(self.op.inputs, kws=[t.params for t in self.op.outputs],
                                         output_limit=len(self.op.outputs),
                                         **self._data.extra_params)


Test patch:
diff --git a/mars/tensor/tests/test_core_execute.py b/mars/tensor/tests/test_core_execute.py
--- a/mars/tensor/tests/test_core_execute.py
+++ b/mars/tensor/tests/test_core_execute.py
@@ -215,7 +215,7 @@ def testCopyAndView(self):
         np.testing.assert_array_equal(a.execute(), npa)
 
         data = np.random.rand(10, 20)
-        a = tensor(data, chunk_size=6)
+        a = tensor(data.copy(), chunk_size=6)
         b = a.copy()
         b[:5] = 10
 
@@ -226,6 +226,19 @@ def testCopyAndView(self):
         np.testing.assert_array_equal(b.execute(), npb)
         np.testing.assert_array_equal(a.execute(), npa)
 
+        a = tensor(data.copy(), chunk_size=6)
+        b = a[:5, :4]
+        c = b.copy()
+        c[0, 0] = 10
+
+        npa = data.copy()
+        npb = npa[:5, :4]
+        npc = npb.copy()
+        npc[0, 0] =10
+
+        np.testing.assert_array_equal(c.execute(), npc)
+        np.testing.assert_array_equal(a.execute(), npa)
+
     def testFlat(self):
         data = np.random.rand(10, 20)
         a = tensor(data, chunk_size=4)
Repository: mars-project/mars

Issue:
[BUG] Web session doesn't work with large data source
**Describe the bug**

The default `max_buffer_size` of tornado HTTP server is 100M, when constructing dataframes from large pandas dataframes it will raise `Content-Length too long`.




Discussion:

Resolving patch:
diff --git a/mars/web/server.py b/mars/web/server.py
--- a/mars/web/server.py
+++ b/mars/web/server.py
@@ -214,6 +214,7 @@ def _try_start_web_server(self):
                     handlers, allow_websocket_origin=['*'],
                     address='0.0.0.0', port=use_port,
                     extra_patterns=extra_patterns,
+                    http_server_kwargs={'max_buffer_size': 2 ** 32},
                 )
                 self._server.start()
                 self._port = use_port


Test patch:
Repository: mars-project/mars

Issue:
[BUG] Tensor shall not be passed into numpy ufuncs when inferring types
**Describe the bug**
When running ``mt.log(a, where=x)``, ``mt.log`` will first call ``infer_type`` to get ``dtype``, and ``x`` whose type is ``mt.Tensor`` is then passed into ``np.log``, which may cause unexpected behaviors.

![image](https://user-images.githubusercontent.com/8284922/74318371-0d426e00-4db8-11ea-873a-2f8e4c767473.png)



Discussion:

Resolving patch:
diff --git a/mars/tensor/utils.py b/mars/tensor/utils.py
--- a/mars/tensor/utils.py
+++ b/mars/tensor/utils.py
@@ -229,7 +229,8 @@ def h(*tensors, **kw):
             args = [make_arg(t) if is_arg(t) else t for t in tensors]
             if reverse:
                 args = args[::-1]
-            np_kw = dict((k, v) for k, v in kw.items() if is_arg(v) and k != 'out')
+            np_kw = dict((k, make_arg(v) if hasattr(v, 'op') else v) for k, v in kw.items()
+                         if is_arg(v) and k != 'out')
 
             dtype = None
             if not any(hasattr(arg, tensor_ufunc)


Test patch:
diff --git a/mars/tensor/arithmetic/tests/test_arithmetic.py b/mars/tensor/arithmetic/tests/test_arithmetic.py
--- a/mars/tensor/arithmetic/tests/test_arithmetic.py
+++ b/mars/tensor/arithmetic/tests/test_arithmetic.py
@@ -261,7 +261,7 @@ def testDtypeFromOut(self):
         t3 = isfinite(x, y)
         self.assertEqual(t3.dtype, y.dtype)
 
-    def testLogWithOutWhere(self):
+    def testLogWithoutWhere(self):
         t1 = ones((3, 4), chunk_size=2)
 
         t2 = log(t1, out=t1)
Repository: mars-project/mars

Issue:
[BUG] `sort_values` failed when input has unknown shape
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

`df.sort_values` may fail due to unknown shape of input dataframe.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

```python
>>> import mars.dataframe as md
>>> df = md.read_csv('/home/xuye.qin/ml-20m/ratings.csv')
>>> df.sort_values(by='rating')
Traceback (most recent call last):
  File "/home/xuye.qin/miniconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 3296, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-4-d04621974535>", line 1, in <module>
    df.sort_values(by='rating')
  File "/tmp/pycharm_project_852/mars/dataframe/sort/sort_values.py", line 181, in dataframe_sort_values
    axis = validate_axis(axis, df)
  File "/tmp/pycharm_project_852/mars/dataframe/utils.py", line 623, in validate_axis
    'object type {}'.format(axis, type(tileable)))
ValueError: No axis named 0 for object type <class 'mars.dataframe.core.DataFrame'>
```



Discussion:

Resolving patch:
diff --git a/mars/dataframe/utils.py b/mars/dataframe/utils.py
--- a/mars/dataframe/utils.py
+++ b/mars/dataframe/utils.py
@@ -613,7 +613,7 @@ def validate_axis(axis, tileable=None):
     illegal = False
     try:
         axis = operator.index(axis)
-        if axis < 0 or (tileable and axis >= tileable.ndim):
+        if axis < 0 or (tileable is not None and axis >= tileable.ndim):
             illegal = True
     except TypeError:
         illegal = True
diff --git a/mars/tensor/base/psrs.py b/mars/tensor/base/psrs.py
--- a/mars/tensor/base/psrs.py
+++ b/mars/tensor/base/psrs.py
@@ -456,9 +456,10 @@ def execute(cls, ctx, op):
                 kth = xp.arange(p - 1, (p - 1) ** 2 + 1, p - 1)
                 a.partition(kth, axis=op.axis)
 
-            select = slice(p - 1, (p - 1) ** 2 + 1, p - 1)
+            select = slice(p, p ** 2 + 1, p)
             slc = (slice(None),) * op.axis + (select,)
-            ctx[op.outputs[0].key] = a[slc]
+            ctx[op.outputs[0].key] = result = a[slc]
+            assert result.shape[op.axis] == p - 1
 
 
 class PSRSShuffle(TensorMapReduceOperand, TensorOperandMixin):


Test patch:
diff --git a/mars/dataframe/tests/test_utils.py b/mars/dataframe/tests/test_utils.py
--- a/mars/dataframe/tests/test_utils.py
+++ b/mars/dataframe/tests/test_utils.py
@@ -376,3 +376,6 @@ def testValidateAxis(self):
 
         with self.assertRaises(ValueError):
             validate_axis(2, df)
+
+        df2 = df[df[0] < 0.5]  # create unknown shape
+        self.assertEqual(validate_axis(0, df2), 0)
Repository: mars-project/mars

Issue:
Allow Mars to use pyarrow 0.17.0
As pyarrow 0.17.0 is released with https://github.com/apache/arrow/pull/6624 merged, we can now use the latest version of pyarrow.


Discussion:

Resolving patch:
diff --git a/mars/_version.py b/mars/_version.py
--- a/mars/_version.py
+++ b/mars/_version.py
@@ -15,7 +15,7 @@
 import subprocess
 import os
 
-version_info = (0, 4, 0, 'b2')
+version_info = (0, 4, 0, 'rc1')
 _num_index = max(idx if isinstance(v, int) else 0
                  for idx, v in enumerate(version_info))
 __version__ = '.'.join(map(str, version_info[:_num_index + 1])) + \
diff --git a/mars/worker/storage/sharedstore.py b/mars/worker/storage/sharedstore.py
--- a/mars/worker/storage/sharedstore.py
+++ b/mars/worker/storage/sharedstore.py
@@ -21,11 +21,19 @@
     import pyarrow
     from pyarrow import plasma
     try:
-        from pyarrow.plasma import PlasmaObjectNonexistent, PlasmaStoreFull
-    except ImportError:
-        from pyarrow.lib import PlasmaObjectNonexistent, PlasmaStoreFull
+        from pyarrow.plasma import PlasmaObjectNotFound
+    except ImportError:  # pragma: no cover
+        try:
+            from pyarrow.plasma import PlasmaObjectNonexistent as PlasmaObjectNotFound
+        except ImportError:
+            from pyarrow.lib import PlasmaObjectNonexistent as PlasmaObjectNotFound
+
+    try:
+        from pyarrow.plasma import PlasmaStoreFull
+    except ImportError:  # pragma: no cover
+        from pyarrow.lib import PlasmaStoreFull
 except ImportError:  # pragma: no cover
-    pyarrow, plasma, PlasmaObjectNonexistent, PlasmaStoreFull = None, None, None, None
+    pyarrow, plasma, PlasmaObjectNotFound, PlasmaStoreFull = None, None, None, None
 
 logger = logging.getLogger(__name__)
 
@@ -145,7 +153,7 @@ def seal(self, session_id, data_key):
         obj_id = self._get_object_id(session_id, data_key)
         try:
             self._plasma_client.seal(obj_id)
-        except PlasmaObjectNonexistent:
+        except PlasmaObjectNotFound:
             self._mapper_ref.delete(session_id, data_key)
             raise KeyError((session_id, data_key))
 


Test patch:
Repository: mars-project/mars

Issue:
[BUG] Dimensions of nsplits and shape of TensorIndex not consistent
When running test_label_propagation.Test.testPredict, an intermediate TensorIndex has shape (nan, 1), however, the nsplit is ((nan,),).


Discussion:

Resolving patch:
diff --git a/mars/tensor/indexing/getitem.py b/mars/tensor/indexing/getitem.py
--- a/mars/tensor/indexing/getitem.py
+++ b/mars/tensor/indexing/getitem.py
@@ -333,7 +333,15 @@ def _calc_order(a, index):
 def _getitem_nocheck(a, item, convert_bool_to_fancy=None):
     index = process_index(a.ndim, item,
                           convert_bool_to_fancy=convert_bool_to_fancy)
-    shape = calc_shape(a.shape, index)
+    if convert_bool_to_fancy is False:
+        # come from __setitem__, the bool index is not converted to fancy index
+        # if multiple bool indexes or bool + fancy indexes exist,
+        # thus the shape will be wrong,
+        # here we just convert when calculating shape,
+        # refer to issue #1282.
+        shape = calc_shape(a.shape, process_index(a.ndim, index))
+    else:
+        shape = calc_shape(a.shape, index)
     tensor_order = _calc_order(a, index)
     op = TensorIndex(dtype=a.dtype, sparse=a.issparse(), indexes=index,
                      create_view=_is_create_view(index))


Test patch:
Repository: mars-project/mars

Issue:
Add docs about reading data from HDFS or writing data into HDFS
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

Docs about reading data from HDFS or writing data into HDFS could be added.


Discussion:

Resolving patch:
diff --git a/mars/dataframe/datasource/read_csv.py b/mars/dataframe/datasource/read_csv.py
--- a/mars/dataframe/datasource/read_csv.py
+++ b/mars/dataframe/datasource/read_csv.py
@@ -341,7 +341,9 @@ def read_csv(path, names=None, sep=',', index_col=None, compression=None, header
     path : str
         Any valid string path is acceptable. The string could be a URL. Valid
         URL schemes include http, ftp, s3, and file. For file URLs, a host is
-        expected. A local file could be: file://localhost/path/to/table.csv.
+        expected. A local file could be: file://localhost/path/to/table.csv,
+        you can alos read from external resources using a URL like:
+        hdfs://localhost:8020/test.csv.
         If you want to pass in a path object, pandas accepts any ``os.PathLike``.
         By file-like object, we refer to objects with a ``read()`` method, such as
         a file handler (e.g. via builtin ``open`` function) or ``StringIO``.
@@ -608,6 +610,8 @@ def read_csv(path, names=None, sep=',', index_col=None, compression=None, header
     --------
     >>> import mars.dataframe as md
     >>> md.read_csv('data.csv')  # doctest: +SKIP
+    >>> # read from HDFS
+    >>> md.read_csv('hdfs://localhost:8020/test.csv')  # doctest: +SKIP
     """
     # infer dtypes and columns
     if isinstance(path, (list, tuple)):


Test patch:
diff --git a/mars/dataframe/datasource/tests/test_hdfs.py b/mars/dataframe/datasource/tests/test_hdfs.py
--- a/mars/dataframe/datasource/tests/test_hdfs.py
+++ b/mars/dataframe/datasource/tests/test_hdfs.py
@@ -17,6 +17,7 @@
 from io import BytesIO
 
 import pandas as pd
+import numpy as np
 
 import mars.dataframe as md
 from mars.tests.core import TestBase
@@ -69,3 +70,14 @@ def testReadCSVExecution(self):
         expected = pd.read_csv(BytesIO(csv_content))
         res = df.to_pandas()
         pd.testing.assert_frame_equal(expected.reset_index(drop=True), res.reset_index(drop=True))
+
+    def testReadParquetExecution(self):
+        test_df = pd.DataFrame({'a': np.arange(10).astype(np.int64, copy=False),
+                                'b': [f's{i}' for i in range(10)],
+                                'c': np.random.rand(10), })
+        with self.hdfs.open(f"{TEST_DIR}/test.parquet", "wb", replication=1) as f:
+            test_df.to_parquet(f, row_group_size=3)
+
+        df = md.read_parquet(f'hdfs://localhost:8020{TEST_DIR}/test.parquet')
+        res = df.to_pandas()
+        pd.testing.assert_frame_equal(res, test_df)
Repository: mars-project/mars

Issue:
[BUG] Setitem for DataFrame leads to a wrong dtypes
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
Add columns for a DataFrame will lead to a wrong dtypes of input DataFrame.

**To Reproduce**
```python
In [1]: import mars.dataframe as md                                                                 

In [2]: a = md.DataFrame({'a':[1,2,3]})                                                             

In [3]: a['new'] = 1                                                                                

In [4]: a.op.inputs                                                                                 
Out[4]: [DataFrame <op=DataFrameDataSource, key=c212164d24d96ed634711c3b97f334cb>]

In [5]: a.op.inputs[0].dtypes                                                                       
Out[5]: 
a      int64
new    int64
dtype: object
```
**Expected behavior**
Input DataFrame's dtypes should have only one column.



Discussion:

Resolving patch:
diff --git a/mars/dataframe/indexing/setitem.py b/mars/dataframe/indexing/setitem.py
--- a/mars/dataframe/indexing/setitem.py
+++ b/mars/dataframe/indexing/setitem.py
@@ -79,7 +79,7 @@ def __call__(self, target: DataFrame, value):
                                           'with different index for now')
 
         index_value = target.index_value
-        dtypes = target.dtypes
+        dtypes = target.dtypes.copy(deep=True)
         dtypes.loc[self._indexes] = value_dtype
         columns_value = parse_index(dtypes.index, store_data=True)
         ret = self.new_dataframe(inputs, shape=(target.shape[0], len(dtypes)),


Test patch:
diff --git a/mars/dataframe/indexing/tests/test_indexing.py b/mars/dataframe/indexing/tests/test_indexing.py
--- a/mars/dataframe/indexing/tests/test_indexing.py
+++ b/mars/dataframe/indexing/tests/test_indexing.py
@@ -634,6 +634,22 @@ def testSeriesGetitem(self):
         self.assertEqual(result2.chunks[0].dtype, data.dtype)
         self.assertTrue(result2.chunks[0].op.labels, [['i2', 'i4']])
 
+    def testSetitem(self):
+        data = pd.DataFrame(np.random.rand(10, 2), columns=['c1', 'c2'])
+        df = md.DataFrame(data, chunk_size=4)
+
+        df['new'] = 1
+        self.assertEqual(df.shape, (10, 3))
+        pd.testing.assert_series_equal(df.inputs[0].dtypes, data.dtypes)
+
+        tiled = df.tiles()
+        self.assertEqual(tiled.chunks[0].shape, (4, 3))
+        pd.testing.assert_series_equal(tiled.inputs[0].dtypes, data.dtypes)
+        self.assertEqual(tiled.chunks[1].shape, (4, 3))
+        pd.testing.assert_series_equal(tiled.inputs[0].dtypes, data.dtypes)
+        self.assertEqual(tiled.chunks[2].shape, (2, 3))
+        pd.testing.assert_series_equal(tiled.inputs[0].dtypes, data.dtypes)
+
     def testResetIndex(self):
         data = pd.DataFrame([('bird',    389.0),
                              ('bird',     24.0),
Repository: mars-project/mars

Issue:
[BUG] `Cannot find serializable class` raised when calling set_label
When calling DataFrame.columns=xxxx in distributed environment, a KeyError is raised with message `Cannot find serializable class for type_id 1517314310`.


Discussion:

Resolving patch:
diff --git a/mars/dataframe/indexing/__init__.py b/mars/dataframe/indexing/__init__.py
--- a/mars/dataframe/indexing/__init__.py
+++ b/mars/dataframe/indexing/__init__.py
@@ -56,6 +56,10 @@ def _install():
     for cls in INDEX_TYPE:
         setattr(cls, 'rename', index_rename)
 
+    # make sure operand is registered
+    from .set_label import DataFrameSetLabel
+    del DataFrameSetLabel
+
 
 _install()
 del _install


Test patch:
diff --git a/mars/tests/core.py b/mars/tests/core.py
--- a/mars/tests/core.py
+++ b/mars/tests/core.py
@@ -36,6 +36,7 @@
 from mars.optimizes.chunk_graph.fuse import Fusion
 from mars.serialize import serializes, deserializes, \
     ProtobufSerializeProvider, JsonSerializeProvider
+from mars.serialize.core import get_serializables
 from mars.utils import lazy_import
 
 try:
@@ -612,6 +613,8 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._raw_chunk_shapes = dict()
         self._tileable_checked = dict()
+        if not hasattr(type(self), '_serializables_snapshot'):
+            type(self)._serializables_snapshot = get_serializables()
 
     @staticmethod
     def _extract_check_options(kw_dict):
@@ -682,6 +685,14 @@ def _update_tileable_and_chunk_shape(self, tileable_graph, chunk_result, failed_
                 self._tileable_checked[n.key] = True
         return super()._update_tileable_and_chunk_shape(tileable_graph, chunk_result, failed_ops)
 
+    def _check_serializable_registration(self):
+        cur_serializables = get_serializables()
+        if len(cur_serializables) == len(self._serializables_snapshot):
+            return
+        unregistered_set = set(cur_serializables.keys()) - set(self._serializables_snapshot.keys())
+        raise AssertionError('Operands %r not registered on initialization'
+                             % ([cur_serializables[k] for k in unregistered_set],))
+
     def execute_tileable(self, tileable, *args, **kwargs):
         self._extract_check_options(kwargs)
 
@@ -695,6 +706,7 @@ def execute_tileable(self, tileable, *args, **kwargs):
             # check returned type
             if kwargs.get('concat', False):
                 self.assert_object_consistent(tileable, result[0])
+        self._check_serializable_registration()
         return result
 
     execute_tensor = execute_tileable
@@ -716,6 +728,7 @@ def execute_tileables(self, tileables, *args, **kwargs):
                     if _check_options['check_nsplits']:
                         self._check_nsplits(tileable)
                     self.assert_object_consistent(tileable, result)
+        self._check_serializable_registration()
         return results
 
     def fetch_tileables(self, tileables, **kw):