internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: wright-group/WrightTools

Issue:
kit function for guessing array plotting params
To be included in our workflow for plotting completed acquisitions, but as a kit function, it is available for e.g. "from" methods and other niche cases.  Analysis should determine guesses for signed/unsigned boolean, perhaps null, limits.

Heuristic based, using information like min/max, 95% window, or similar.




Discussion:

Resolving patch:
diff --git a/WrightTools/kit/_array.py b/WrightTools/kit/_array.py
--- a/WrightTools/kit/_array.py
+++ b/WrightTools/kit/_array.py
@@ -27,6 +27,7 @@
     "valid_index",
     "mask_reduce",
     "enforce_mask_shape",
+    "guess_signed",
 ]
 
 
@@ -432,3 +433,38 @@ def enforce_mask_shape(mask, shape):
     """
     red = tuple([i for i in range(len(shape)) if shape[i] == 1])
     return mask.max(axis=red, keepdims=True)
+
+
+def guess_signed(chan, tol=1e-1):
+    """guess whether or not a channel is signed by examining min and max values.
+
+    Parameters
+    -------------
+    chan : array-like
+        Input channel or array
+    tol : float (optional)
+        Tolerance value used to judge signed. `tol` should be much less than 1. To prefer signed guesses, use negative tolerance values.
+
+    Returns
+    -------
+    guess : bool
+        True if the data seems signed and False otherwise.
+    """
+
+    maxc = chan.max()
+    minc = chan.min()
+    from ..data import Channel
+
+    if isinstance(chan, Channel):
+        null = chan.null
+    else:
+        null = 0
+
+    # avoid zero division for comparison
+    bottom = np.abs(maxc - null) + np.abs(minc - null)
+    if not bottom:  # (maxc-null)=-(minc-null)
+        return True
+
+    comparison = np.abs(maxc + minc - 2 * null) / bottom
+    # should be < 1 if signed
+    return comparison < 1 - tol


Test patch:
diff --git a/tests/kit/signed.py b/tests/kit/signed.py
new file mode 100644
--- /dev/null
+++ b/tests/kit/signed.py
@@ -0,0 +1,30 @@
+"""Test guess_signed"""
+
+# --- import -------------------------------------------------------------------------------------
+
+
+import numpy as np
+import WrightTools as wt
+
+
+# --- test ---------------------------------------------------------------------------------------
+
+
+def test_many_ranges():
+    for minmax, signed in [
+        ([-0.05, 1], False),
+        ([-1, 0.05], False),
+        ([-1, 1], True),
+        ([0, 0], True),
+        ([0, -1], False),
+        ([-1, 0.5], True),
+    ]:
+        assert wt.kit.guess_signed(np.array(minmax)) == signed
+
+
+def test_channel():
+    d = wt.Data()
+    chan = d.create_channel("chan", values=np.linspace(3, 4, 16).reshape(4, 4))
+    assert wt.kit.guess_signed(chan) == False
+    chan.null = 3.5
+    assert wt.kit.guess_signed(chan)
Repository: wright-group/WrightTools

Issue:
shift supported Python 3 versions
Since users are increasingly relying on 3.10 and 3.11, I propose we move testing from 3.7-9 to 3.8-11.



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -74,9 +74,10 @@ def read(fname):
         "Framework :: Matplotlib",
         "Natural Language :: English",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Topic :: Scientific/Engineering",
     ],
 )


Test patch:
Repository: wright-group/WrightTools

Issue:
interact2D: more documentation
* show usage in docstring: make sure users know to avoid garbage collection (`out = wt.artists.interact2D`)
* gif for docs showing interactivity
* note important matplotlib settings (e.g. do not use inline plotting)


Discussion:

Resolving patch:
diff --git a/WrightTools/artists/_interact.py b/WrightTools/artists/_interact.py
--- a/WrightTools/artists/_interact.py
+++ b/WrightTools/artists/_interact.py
@@ -203,6 +203,31 @@ def interact2D(
         uniform grids.  Default is False.
     verbose : boolean (optional)
         Toggle talkback. Default is True.
+
+    Returns
+    -------
+    out : types.SimpleNamespace
+        container for important interactive elements of the plot.
+
+        Properties
+        ----------
+        image : 2D artist object
+            The artist of the 2D plot.  Type is either matplotlib.collections.QuadMesh (use_imshow=False) or matplotlib.image.AxesImage (use_imshow=True)
+        sliders : dict (key[name] = value[matplotlib.Widgets.Slider])
+            The sliders.
+        crosshairs: list of horizontal and vertical crosshair
+            crosshairs are lists of matplotlib.lines.Line2D
+        radio : matplotlib.Widgets.RadioButtons
+            radio button for local/global normalization
+        colorbar : matplotlib.colorbar.ColorbarBase
+            The 2D colorbar object
+
+    Usage
+    -----
+    When calling, always store the returned arguments to a variable:
+    >>> out = WrightTools.artists.interact2D(...)
+
+    This prevents interactive buttons from turning off due to python's garbage collection.
     """
     # avoid changing passed data object
     data = data.copy()
@@ -556,4 +581,11 @@ def update_key_press(info):
     for slider in sliders.values():
         slider.on_changed(update_slider)
 
-    return obj2D, sliders, crosshair_hline, crosshair_vline, radio, colorbar
+    out = SimpleNamespace(
+        image=obj2D,
+        sliders=sliders,
+        crosshairs=[crosshair_hline, crosshair_vline],
+        radio=radio,
+        colorbar=colorbar,
+    )
+    return out


Test patch:
Repository: databricks/koalas

Issue:
Fix pypi description
See https://pypi.org/project/databricks-koalas/

It just pulls in our entire README in markdown format.



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -18,24 +18,46 @@
 
 from setuptools import setup
 
+DESCRIPTION = "Pandas DataFrame API on Apache Spark"
 
-install_requires = [
-    'pandas>=0.23',
-    'decorator',
-    'pyarrow>=0.10,<0.11',  # See https://github.com/databricks/spark-pandas/issues/26
-]
+LONG_DESCRIPTION = """
+Koalas makes data scientists more productive when interacting with big data,
+by augmenting Apache Spark's Python DataFrame API to be compatible with
+Pandas'.
+
+Pandas is the de facto standard (single-node) dataframe implementation in
+Python, while Spark is the de facto standard for big data processing.
+With this package, data scientists can:
+
+- Be immediately productive with Spark, with no learning curve, if one
+  is already familiar with Pandas.
+- Have a single codebase that works both with Pandas (tests, smaller datasets)
+  and with Spark (distributed datasets).
+"""
 
 setup(
-    name='databricks-koalas',
+    name='koalas',
     version='0.0.6',
     packages=['databricks', 'databricks.koalas', 'databricks.koalas.dask',
               'databricks.koalas.missing'],
     extras_require={
         'spark': ['pyspark>=2.4.0'],
     },
-    install_requires=install_requires,
-    author="Timothy Hunter",
-    author_email="tim@databricks.com",
+    python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*',
+    install_requires=[
+        'pandas>=0.23',
+        'decorator',
+        'pyarrow>=0.10,<0.11',  # See https://github.com/databricks/spark-pandas/issues/26
+    ],
+    maintainer="Databricks",
+    maintainer_email="pandas-on-spark@databricks.com",
     license='http://www.apache.org/licenses/LICENSE-2.0',
-    long_description=open('README.md').read(),
+    url="https://github.com/databricks/spark-pandas",
+    project_urls={
+        'Bug Tracker': 'https://github.com/databricks/spark-pandas/issues',
+        # 'Documentation': '',
+        'Source Code': 'https://github.com/databricks/spark-pandas'
+    },
+    description=DESCRIPTION,
+    long_description=LONG_DESCRIPTION,
 )


Test patch:
Repository: databricks/koalas

Issue:
How to call dev/_make_missing_functions.py?
```
rxin @ C02XT0W6JGH5 : ~/workspace/spark-pandas (master) 
> dev/_make_missing_functions.py 
Traceback (most recent call last):
  File "dev/_make_missing_functions.py", line 22, in <module>
    from databricks.koala.frame import PandasLikeDataFrame
ImportError: No module named databricks.koala.frame
```

Do I need to install koala first? We should add documentation to CONTRIBUTING.md. It'd also be best if this runs against the existing code base, rather than a system-wide installed Koala.



Discussion:

Resolving patch:
diff --git a/dev/_make_missing_functions.py b/dev/_make_missing_functions.py
--- a/dev/_make_missing_functions.py
+++ b/dev/_make_missing_functions.py
@@ -15,6 +15,12 @@
 # limitations under the License.
 #
 
+"""
+A script to generate the missing function stubs. Before running this,
+make sure you install koalas from the current checkout by running:
+pip install -e .
+"""
+
 import inspect
 
 import pandas as pd


Test patch:
Repository: databricks/koalas

Issue:
Show pandas style Table of Contents on the left side in docs
Right now our docs show a weird Table of Contents for only a section, rather than the entire doc. Can we fix it so it shows the Table of Contents of the entire docs, e.g. start from the top level?

<img width="647" alt="Screen Shot 2019-04-23 at 4 40 38 PM" src="https://user-images.githubusercontent.com/323388/56622865-9351b600-65e6-11e9-98b3-7930660b1c93.png">




Discussion:

Resolving patch:
diff --git a/docs/source/conf.py b/docs/source/conf.py
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -68,13 +68,16 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'nature'
+html_theme = 'nature_with_gtoc'
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
+# Add any paths that contain custom themes here, relative to this directory.
+html_theme_path = ['themes']
+
 # If false, no index is generated.
 html_use_index = False
 


Test patch:
Repository: databricks/koalas

Issue:
Document all the methods in Metadata
There are a bunch of methods like index_info, index_fields. It's pretty difficult to figure out what they do. We should just add some basic docstring comments.

@ueshin you are probably the best person to take this since you created the file.



Discussion:
@rxin sure, I'll take this too.

Resolving patch:
diff --git a/databricks/koalas/metadata.py b/databricks/koalas/metadata.py
--- a/databricks/koalas/metadata.py
+++ b/databricks/koalas/metadata.py
@@ -24,7 +24,11 @@
 
 class Metadata(object):
     """
-    Manages column names and index information
+    Manages column names and index information.
+
+    :ivar _column_fields: list of the Spark field names to be seen as columns in Koalas DataFrame.
+    :ivar _index_info: list of pair holding the Spark field names for indexes,
+                       and the index name to be seen in Koalas DataFrame.
     """
 
     def __init__(self, column_fields, index_info=None):
@@ -46,27 +50,38 @@ def __init__(self, column_fields, index_info=None):
 
     @property
     def column_fields(self):
+        """ Returns the managed column field names. """
         return self._column_fields
 
     @property
     def index_info(self):
+        """ Return the managed index information. """
         return self._index_info
 
     @property
     def index_fields(self):
+        """ Returns the managed index field names. """
         return [index_field for index_field, _ in self._index_info]
 
     @property
     def index_names(self):
+        """ Return the managed index names. """
         return [name for _, name in self._index_info]
 
     @property
     def all_fields(self):
+        """ Return all the field names including index field names. """
         index_fields = self.index_fields
         return index_fields + [field for field in self._column_fields
                                if field not in index_fields]
 
     def copy(self, column_fields=None, index_info=None):
+        """ Copy the metadata.
+
+        :param column_fields: the new column field names. If None, then the original ones are used.
+        :param index_info: the new index information. If None, then the original one is used.
+        :return: the copied metadata.
+        """
         if column_fields is None:
             column_fields = self._column_fields
         if index_info is None:
@@ -75,6 +90,11 @@ def copy(self, column_fields=None, index_info=None):
 
     @staticmethod
     def from_pandas(pdf):
+        """ Create a metadata from pandas DataFrame.
+
+        :param pdf: :class:`pd.DataFrame`
+        :return: the created metadata
+        """
         column_fields = [str(col) for col in pdf.columns]
         index = pdf.index
         if isinstance(index, pd.MultiIndex):


Test patch:
Repository: databricks/koalas

Issue:
Monkey patch Spark's DataFrame to add to_koalas method
Create a simple and convenient way to turn a Spark DataFrame into a Koalas DataFrame.



Discussion:

Resolving patch:
diff --git a/databricks/koalas/__init__.py b/databricks/koalas/__init__.py
--- a/databricks/koalas/__init__.py
+++ b/databricks/koalas/__init__.py
@@ -42,3 +42,20 @@ def assert_pyspark_version():
 
 __all__ = ['read_csv', 'read_parquet', 'to_datetime', 'from_pandas',
            'get_dummies', 'DataFrame', 'Series']
+
+
+def _auto_patch():
+    import os
+    import logging
+    # Autopatching is on by default.
+    x = os.getenv("SPARK_KOALAS_AUTOPATCH", "true")
+    if x.lower() in ("true", "1", "enabled"):
+        logger = logging.getLogger('spark')
+        logger.info("Patching spark automatically. You can disable it by setting "
+                    "SPARK_KOALAS_AUTOPATCH=false in your environment")
+
+    from pyspark.sql import dataframe as df
+    df.DataFrame.to_koalas = DataFrame.to_koalas
+
+
+_auto_patch()
diff --git a/databricks/koalas/frame.py b/databricks/koalas/frame.py
--- a/databricks/koalas/frame.py
+++ b/databricks/koalas/frame.py
@@ -303,6 +303,55 @@ def notnull(self):
 
     notna = notnull
 
+    def to_koalas(self):
+        """
+        Converts the existing DataFrame into a Koalas DataFrame.
+
+        This method is monkey-patched into Spark's DataFrame and can be used
+        to convert a Spark DataFrame into a Koalas DataFrame. If running on
+        an existing Koalas DataFrame, the method returns itself.
+
+        If a Koalas DataFrame is converted to a Spark DataFrame and then back
+        to Koalas, it will lose the index information and the original index
+        will be turned into a normal column.
+
+        See Also
+        --------
+        DataFrame.to_spark
+
+        Examples
+        --------
+        >>> df = ks.DataFrame({'col1': [1, 2], 'col2': [3, 4]})
+        >>> df
+           col1  col2
+        0     1     3
+        1     2     4
+
+        >>> spark_df = df.to_spark()
+        >>> spark_df
+        DataFrame[__index_level_0__: bigint, col1: bigint, col2: bigint]
+
+        >>> kdf = spark_df.to_koalas()
+        >>> kdf
+           __index_level_0__  col1  col2
+        0                  0     1     3
+        1                  1     2     4
+        """
+        if isinstance(self, DataFrame):
+            return self
+        else:
+            return DataFrame(self)
+
+    def to_spark(self):
+        """
+        Return the current DataFrame as a Spark DataFrame.
+
+        See Also
+        --------
+        DataFrame.to_koalas
+        """
+        return self._sdf
+
     @derived_from(spark.DataFrame)
     def toPandas(self):
         sdf = self._sdf.select(['`{}`'.format(name) for name in self._metadata.all_fields])


Test patch:
Repository: databricks/koalas

Issue:
unique function is broken
```
df = ks.DataFrame({"Person": ['a', 'b', 'b', 'c']})
df
  Person
0      a
1      b
2      b
3      c

df.unique()
  Person
2      b
3      c
0      a
1      b
```


Discussion:
Seems like Pandas DataFrame doesn't have `unique`:

```python
>>> import pandas as pd
>>> pd.DataFrame({"Person": ['a', 'b', 'b', 'c']})
  Person
0      a
1      b
2      b
3      c
>>> df = pd.DataFrame({"Person": ['a', 'b', 'b', 'c']})
>>> df.unique()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python3.7/site-packages/pandas/core/generic.py", line 4376, in __getattr__
    return object.__getattribute__(self, name)
AttributeError: 'DataFrame' object has no attribute 'unique'
>>> df.Person.unique()
array(['a', 'b', 'c'], dtype=object)
```

but series has it. Let me target to fix Series one.
Yea maybe we should remove DataFrame.unique then and only keep Series.unique.


Resolving patch:
diff --git a/databricks/koalas/frame.py b/databricks/koalas/frame.py
--- a/databricks/koalas/frame.py
+++ b/databricks/koalas/frame.py
@@ -1160,10 +1160,6 @@ def count(self):
         """
         return self._reduce_for_stat_function(_Frame._count_expr)
 
-    def unique(self):
-        sdf = self._sdf
-        return DataFrame(spark.DataFrame(sdf._jdf.distinct(), sdf.sql_ctx), self._metadata.copy())
-
     def drop(self, labels=None, axis=1, columns: Union[str, List[str]] = None):
         """
         Drop specified labels from columns.
diff --git a/databricks/koalas/series.py b/databricks/koalas/series.py
--- a/databricks/koalas/series.py
+++ b/databricks/koalas/series.py
@@ -491,9 +491,38 @@ def dropna(self, axis=0, inplace=False, **kwargs):
     def head(self, n=5):
         return _col(self.to_dataframe().head(n))
 
+    # TODO: Categorical type isn't supported (due to PySpark's limitation) and
+    # some doctests related with timestamps were not added.
     def unique(self):
-        # Pandas wants a series/array-like object
-        return _col(self.to_dataframe().unique())
+        """
+        Return unique values of Series object.
+
+        Uniques are returned in order of appearance. Hash table-based unique,
+        therefore does NOT sort.
+
+        .. note:: This method returns newly creased Series whereas Pandas returns
+                  the unique values as a NumPy array.
+
+        Returns
+        -------
+        Returns the unique values as a Series.
+
+        See Examples section.
+
+        Examples
+        --------
+        >>> ks.Series([2, 1, 3, 3], name='A').unique()
+        0    1
+        1    3
+        2    2
+        Name: A, dtype: int64
+
+        >>> ks.Series([pd.Timestamp('2016-01-01') for _ in range(3)]).unique()
+        0   2016-01-01
+        Name: 0, dtype: datetime64[ns]
+        """
+        sdf = self.to_dataframe()._sdf
+        return _col(DataFrame(sdf.select(self._scol).distinct()))
 
     # TODO: Update Documentation for Bins Parameter when its supported
     def value_counts(self, normalize=False, sort=True, ascending=False, bins=None, dropna=True):


Test patch:
Repository: databricks/koalas

Issue:
Remove DataFrame.__iter__
The current implementation is too error-prone in big data:

```
    def __iter__(self):
        return self.toPandas().__iter__()
```

It will just blow up with large amount of data.



Discussion:
cc @thunterdb since you added this one ...

For now I think we should just remove it. We can add it back when we have a better way to do this. One way to do this is through toLocalIterator, but it's still a bad practice in general.


Resolving patch:
diff --git a/databricks/koalas/frame.py b/databricks/koalas/frame.py
--- a/databricks/koalas/frame.py
+++ b/databricks/koalas/frame.py
@@ -2663,9 +2663,6 @@ def __getattr__(self, key: str) -> Any:
                 return partial(property_or_func, self)
         return Series(self._sdf.__getattr__(key), anchor=self, index=self._metadata.index_map)
 
-    def __iter__(self):
-        return self.toPandas().__iter__()
-
     def __len__(self):
         return self._sdf.count()
 


Test patch:
Repository: databricks/koalas

Issue:
isna() is not working with UDFs
I am experiencing the following issue with to_timestamp. Here is a reproduction:

```py
@ks.pandas_wraps
def f(x) -> ks.Col[int]:
  return 2 * x

df = ks.DataFrame({"x":[1, None]})
f(df["x"]).isna()
```

It works with pandas but triggers the following error in spark: `<repr(<databricks.koalas.series.Series at 0x7fd2d0777438>) failed: KeyError: '(CAST(clean_fun(x) AS INT) AS `f(x)` IS NULL)'>`


Discussion:
The cause seems to be https://issues.apache.org/jira/browse/SPARK-18502. Let me try to find a workaround

Resolving patch:
diff --git a/databricks/koalas/base.py b/databricks/koalas/base.py
--- a/databricks/koalas/base.py
+++ b/databricks/koalas/base.py
@@ -322,12 +322,12 @@ def isnull(self):
         0    False
         1    False
         2     True
-        Name: ((0 IS NULL) OR isnan(0)), dtype: bool
+        Name: 0, dtype: bool
         """
         if isinstance(self.spark_type, (FloatType, DoubleType)):
-            return self._with_new_scol(self._scol.isNull() | F.isnan(self._scol))
+            return self._with_new_scol(self._scol.isNull() | F.isnan(self._scol)).alias(self.name)
         else:
-            return self._with_new_scol(self._scol.isNull())
+            return self._with_new_scol(self._scol.isNull()).alias(self.name)
 
     isna = isnull
 
@@ -360,9 +360,9 @@ def notnull(self):
         0     True
         1     True
         2    False
-        Name: (NOT ((0 IS NULL) OR isnan(0))), dtype: bool
+        Name: 0, dtype: bool
         """
-        return ~self.isnull()
+        return (~self.isnull()).alias(self.name)
 
     notna = notnull
 


Test patch:
diff --git a/databricks/koalas/tests/test_series.py b/databricks/koalas/tests/test_series.py
--- a/databricks/koalas/tests/test_series.py
+++ b/databricks/koalas/tests/test_series.py
@@ -386,3 +386,16 @@ def test_map(self):
         self.assertEqual(
             repr(kser.map(d)),
             repr(pser.map(d).rename(0)))
+
+    def test_pandas_wraps(self):
+        # This test checks the return column name of `isna()`. Previously it returned the column
+        # name as its internal expression which contains, for instance, '`f(x)`' in the middle of
+        # column name which currently cannot be recognized in PySpark.
+        @koalas.pandas_wraps
+        def f(x) -> koalas.Col[int]:
+            return 2 * x
+
+        df = koalas.DataFrame({"x": [1, None]})
+        self.assert_eq(
+            f(df["x"]).isna(),
+            pd.Series([False, True]).rename("f(x)"))