nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: dask/dask
Pull request #8969 (2022-04-21 22:47:12)

parse_timedelta option to enforce explicit unit
Used by dask/distributed


Patch:
diff --git a/dask/utils.py b/dask/utils.py
index f3d216afd..9486ad0f2 100644
--- a/dask/utils.py
+++ b/dask/utils.py
@@ -18,7 +18,7 @@ from importlib import import_module
 from numbers import Integral, Number
 from operator import add
 from threading import Lock
-from typing import Any, ClassVar, TypeVar, overload
+from typing import Any, ClassVar, Literal, TypeVar, overload
 from weakref import WeakValueDictionary
 
 import tlz as toolz
@@ -1587,18 +1587,27 @@ timedelta_sizes.update({k.upper(): v for k, v in timedelta_sizes.items()})
 
 
 @overload
-def parse_timedelta(s: None, default: str = "seconds") -> None:
+def parse_timedelta(s: None, default: str | Literal[False] = "seconds") -> None:
     ...
 
 
 @overload
-def parse_timedelta(s: str | float | timedelta, default: str = "seconds") -> float:
+def parse_timedelta(
+    s: str | float | timedelta, default: str | Literal[False] = "seconds"
+) -> float:
     ...
 
 
 def parse_timedelta(s, default="seconds"):
     """Parse timedelta string to number of seconds
 
+    Parameters
+    ----------
+    s : str, float, timedelta, or None
+    default: str or False, optional
+        Unit of measure if s  does not specify one. Defaults to seconds.
+        Set to False to require s to explicitly specify its own unit.
+
     Examples
     --------
     >>> from datetime import timedelta
@@ -1630,6 +1639,10 @@ def parse_timedelta(s, default="seconds"):
 
     prefix = s[:index]
     suffix = s[index:] or default
+    if suffix is False:
+        raise ValueError(f"Missing time unit: {s}")
+    if not isinstance(suffix, str):
+        raise TypeError(f"default must be str or False, got {default!r}")
 
     n = float(prefix)
 


Test patch:
diff --git a/dask/tests/test_utils.py b/dask/tests/test_utils.py
index 7349628a2..9776b901a 100644
--- a/dask/tests/test_utils.py
+++ b/dask/tests/test_utils.py
@@ -620,6 +620,14 @@ def test_parse_timedelta():
     assert parse_timedelta("1", default="ms") == 0.001
     assert parse_timedelta(1, default="ms") == 0.001
 
+    assert parse_timedelta("1ms", default=False) == 0.001
+    with pytest.raises(ValueError):
+        parse_timedelta(1, default=False)
+    with pytest.raises(ValueError):
+        parse_timedelta("1", default=False)
+    with pytest.raises(TypeError):
+        parse_timedelta("1", default=None)
+
 
 def test_is_arraylike():
     np = pytest.importorskip("numpy")
Repository: dask/dask
Pull request #10487 (2023-08-31 14:40:13)

Special handling for None in DASK_ environment variables
- Complements https://github.com/dask/distributed/pull/8144

We use `ast.literal_eval` to parse environment variables. While that works fine for numbers and even for complicated nested python objects, it's not user friendly for None, where users are likely to get confused with capitalization (keys are all uppercase) and with yaml/json where the value is null.

CC @ntabris 


Patch:
diff --git a/dask/config.py b/dask/config.py
index 46f7ef60d..2f2c837ef 100644
--- a/dask/config.py
+++ b/dask/config.py
@@ -260,7 +260,10 @@ def collect_env(env: Mapping[str, str] | None = None) -> dict:
             try:
                 d[varname] = ast.literal_eval(value)
             except (SyntaxError, ValueError):
-                d[varname] = value
+                if value.lower() in ("none", "null"):
+                    d[varname] = None
+                else:
+                    d[varname] = value
 
     result: dict = {}
     set(d, config=result)


Test patch:
diff --git a/dask/tests/test_config.py b/dask/tests/test_config.py
index aa59f8660..ab62cb8a9 100644
--- a/dask/tests/test_config.py
+++ b/dask/tests/test_config.py
@@ -227,10 +227,33 @@ def test_env():
     assert res == expected
 
 
+def test_env_none_values():
+    env = {
+        "DASK_A": "None",
+        "DASK_B": "NONE",
+        "DASK_C": "none",
+        "DASK_D": "Null",
+        "DASK_E": "NULL",
+        "DASK_F": "null",
+    }
+
+    expected = {
+        "a": None,
+        "b": None,
+        "c": None,
+        "d": None,
+        "e": None,
+        "f": None,
+    }
+
+    res = collect_env(env)
+    assert res == expected
+
+
 def test_collect():
     a = {"x": 1, "y": {"a": 1}}
     b = {"x": 2, "z": 3, "y": {"b": 2}}
-    env = {"DASK_W": 4}
+    env = {"DASK_W": "4"}
 
     expected = {"w": 4, "x": 2, "y": {"a": 1, "b": 2}, "z": 3}
Repository: dask/dask
Pull request #10488 (2023-08-31 21:33:56)

Skip failing tests for ``pandas=2.1.0``
- [ ] Closes #xxxx
- [ ] Tests added / passed
- [ ] Passes `pre-commit run --all-files`



Patch:
diff --git a/dask/dataframe/_compat.py b/dask/dataframe/_compat.py
index bf8bfcfcc..9da0af718 100644
--- a/dask/dataframe/_compat.py
+++ b/dask/dataframe/_compat.py
@@ -17,6 +17,7 @@ PANDAS_GE_200 = PANDAS_VERSION.major >= 2
 PANDAS_GE_201 = PANDAS_VERSION.release >= (2, 0, 1)
 PANDAS_GE_202 = PANDAS_VERSION.release >= (2, 0, 2)
 PANDAS_GE_210 = PANDAS_VERSION.release >= (2, 1, 0)
+PANDAS_GE_211 = PANDAS_VERSION.release >= (2, 1, 1)
 
 import pandas.testing as tm
 
diff --git a/dask/dataframe/io/parquet/arrow.py b/dask/dataframe/io/parquet/arrow.py
index c17b33e1b..a1c237f30 100644
--- a/dask/dataframe/io/parquet/arrow.py
+++ b/dask/dataframe/io/parquet/arrow.py
@@ -22,7 +22,6 @@ from pyarrow import fs as pa_fs
 import dask
 from dask.base import tokenize
 from dask.core import flatten
-from dask.dataframe._compat import check_observed_deprecation
 from dask.dataframe.backends import pyarrow_schema_dispatch
 from dask.dataframe.io.parquet.utils import (
     Engine,
@@ -130,8 +129,7 @@ def _write_partitioned(
 
     md_list = []
     partition_keys = partition_keys[0] if len(partition_keys) == 1 else partition_keys
-    with check_observed_deprecation():
-        gb = data_df.groupby(partition_keys, dropna=False)
+    gb = data_df.groupby(partition_keys, dropna=False, observed=False)
     for keys, subgroup in gb:
         if not isinstance(keys, tuple):
             keys = (keys,)


Test patch:
diff --git a/dask/dataframe/tests/test_merge_column_and_index.py b/dask/dataframe/tests/test_merge_column_and_index.py
index 1a9503206..2b5aa745d 100644
--- a/dask/dataframe/tests/test_merge_column_and_index.py
+++ b/dask/dataframe/tests/test_merge_column_and_index.py
@@ -5,6 +5,7 @@ import pandas as pd
 import pytest
 
 import dask.dataframe as dd
+from dask.dataframe._compat import PANDAS_GE_210, PANDAS_GE_211
 from dask.dataframe.utils import assert_eq
 
 
@@ -229,6 +230,7 @@ def test_merge_known_to_double_bcast_left(
     result.head(1)
 
 
+@pytest.mark.skipif(PANDAS_GE_210 and not PANDAS_GE_211, reason="breaks on 2.1.0")
 @pytest.mark.parametrize("repartition", [None, 4])
 def test_merge_column_with_nulls(repartition):
     # See: https://github.com/dask/dask/issues/7558
diff --git a/dask/dataframe/tests/test_multi.py b/dask/dataframe/tests/test_multi.py
index d15cea1c7..ba9ee8d0a 100644
--- a/dask/dataframe/tests/test_multi.py
+++ b/dask/dataframe/tests/test_multi.py
@@ -14,6 +14,7 @@ from dask.dataframe._compat import (
     PANDAS_GE_150,
     PANDAS_GE_200,
     PANDAS_GE_210,
+    PANDAS_GE_211,
     tm,
 )
 from dask.dataframe.core import _Frame
@@ -1364,6 +1365,7 @@ def test_merge_by_index_patterns(how, shuffle_method):
             )
 
 
+@pytest.mark.skipif(PANDAS_GE_210 and not PANDAS_GE_211, reason="breaks on 2.1.0")
 @pytest.mark.slow
 @pytest.mark.parametrize("how", ["inner", "outer", "left", "right"])
 def test_join_by_index_patterns(how, shuffle_method):
Repository: dask/dask
Pull request #10640 (2023-11-21 16:44:29)

Add tokenize NA functionality
Ran into this in dask-expr


Patch:
diff --git a/dask/base.py b/dask/base.py
index 044beb7ac..4f3eccf15 100644
--- a/dask/base.py
+++ b/dask/base.py
@@ -1194,6 +1194,10 @@ def register_pandas():
     def normalize_period_dtype(dtype):
         return normalize_token(dtype.name)
 
+    @normalize_token.register(type(pd.NA))
+    def normalize_na(dtype):
+        return pd.NA
+
 
 @normalize_token.register_lazy("numpy")
 def register_numpy():


Test patch:
diff --git a/dask/tests/test_base.py b/dask/tests/test_base.py
index 776c360ad..d1af7fb2e 100644
--- a/dask/tests/test_base.py
+++ b/dask/tests/test_base.py
@@ -354,6 +354,11 @@ def test_tokenize_pandas_extension_array():
         assert tokenize(arr) == tokenize(arr)
 
 
+@pytest.mark.skipif("not dd")
+def test_tokenize_na():
+    assert tokenize(pd.NA) == tokenize(pd.NA)
+
+
 @pytest.mark.skipif("not pd")
 def test_tokenize_pandas_index():
     idx = pd.Index(["a", "b"])
Repository: dask/dask
Pull request #10643 (2023-11-22 11:12:28)

Tokenize pandas offsets deterministically
Another thing we need for dask-expr



Patch:
diff --git a/dask/base.py b/dask/base.py
index 4f3eccf15..c015c4b70 100644
--- a/dask/base.py
+++ b/dask/base.py
@@ -1195,9 +1195,13 @@ def register_pandas():
         return normalize_token(dtype.name)
 
     @normalize_token.register(type(pd.NA))
-    def normalize_na(dtype):
+    def normalize_na(na):
         return pd.NA
 
+    @normalize_token.register(pd.offsets.BaseOffset)
+    def normalize_offset(offset):
+        return [offset.n, offset.name]
+
 
 @normalize_token.register_lazy("numpy")
 def register_numpy():


Test patch:
diff --git a/dask/tests/test_base.py b/dask/tests/test_base.py
index d1af7fb2e..c5217f697 100644
--- a/dask/tests/test_base.py
+++ b/dask/tests/test_base.py
@@ -354,11 +354,22 @@ def test_tokenize_pandas_extension_array():
         assert tokenize(arr) == tokenize(arr)
 
 
-@pytest.mark.skipif("not dd")
+@pytest.mark.skipif("not pd")
 def test_tokenize_na():
     assert tokenize(pd.NA) == tokenize(pd.NA)
 
 
+@pytest.mark.skipif("not pd")
+def test_tokenize_offset():
+    for offset in [
+        pd.offsets.Second(1),
+        pd.offsets.MonthBegin(2),
+        pd.offsets.Day(1),
+        pd.offsets.BQuarterEnd(2),
+    ]:
+        assert tokenize(offset) == tokenize(offset)
+
+
 @pytest.mark.skipif("not pd")
 def test_tokenize_pandas_index():
     idx = pd.Index(["a", "b"])
Repository: dask/dask
Pull request #10653 (2023-11-27 22:32:10)

Add ``network`` marker to ``test_pyarrow_filesystem_option_real_data``
Minor followup to #10590 (see: https://github.com/dask/dask/pull/10590/files#r1405309723)


Patch:
diff --git a/continuous_integration/gpuci/axis.yaml b/continuous_integration/gpuci/axis.yaml
index c66f47166..32c44da26 100644
--- a/continuous_integration/gpuci/axis.yaml
+++ b/continuous_integration/gpuci/axis.yaml
@@ -3,7 +3,7 @@ PYTHON_VER:
 - "3.10"
 
 CUDA_VER:
-- "11.5.2"
+- "11.8.0"
 
 LINUX_VER:
 - ubuntu20.04
diff --git a/dask/base.py b/dask/base.py
index 4f3eccf15..c015c4b70 100644
--- a/dask/base.py
+++ b/dask/base.py
@@ -1195,9 +1195,13 @@ def register_pandas():
         return normalize_token(dtype.name)
 
     @normalize_token.register(type(pd.NA))
-    def normalize_na(dtype):
+    def normalize_na(na):
         return pd.NA
 
+    @normalize_token.register(pd.offsets.BaseOffset)
+    def normalize_offset(offset):
+        return [offset.n, offset.name]
+
 
 @normalize_token.register_lazy("numpy")
 def register_numpy():


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
index 518145cc6..0d4dd39d7 100644
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -4741,6 +4741,7 @@ def test_pyarrow_filesystem_option(tmp_path, fs):
 
 
 @PYARROW_MARK
+@pytest.mark.network
 @pytest.mark.slow
 def test_pyarrow_filesystem_option_real_data():
     # See: https://github.com/dask/dask/pull/10590
diff --git a/dask/tests/test_base.py b/dask/tests/test_base.py
index d1af7fb2e..c5217f697 100644
--- a/dask/tests/test_base.py
+++ b/dask/tests/test_base.py
@@ -354,11 +354,22 @@ def test_tokenize_pandas_extension_array():
         assert tokenize(arr) == tokenize(arr)
 
 
-@pytest.mark.skipif("not dd")
+@pytest.mark.skipif("not pd")
 def test_tokenize_na():
     assert tokenize(pd.NA) == tokenize(pd.NA)
 
 
+@pytest.mark.skipif("not pd")
+def test_tokenize_offset():
+    for offset in [
+        pd.offsets.Second(1),
+        pd.offsets.MonthBegin(2),
+        pd.offsets.Day(1),
+        pd.offsets.BQuarterEnd(2),
+    ]:
+        assert tokenize(offset) == tokenize(offset)
+
+
 @pytest.mark.skipif("not pd")
 def test_tokenize_pandas_index():
     idx = pd.Index(["a", "b"])
Repository: dask/dask
Pull request #4774 (2019-05-04 15:07:04)

Add core dask config file
- [ ] Tests added / passed
- [ ] Passes `flake8 dask`

cc @jcrist 

mrocklin: See also https://github.com/dask/distributed/pull/2654
jcrist: Seems fine to me. Fixes #4609.

Patch:
diff --git a/dask/config.py b/dask/config.py
index c9eabf08f..a55541455 100644
--- a/dask/config.py
+++ b/dask/config.py
@@ -497,3 +497,14 @@ def expand_environment_variables(config):
 
 
 refresh()
+
+
+if yaml:
+    fn = os.path.join(os.path.dirname(__file__), "dask.yaml")
+    ensure_file(source=fn)
+
+    with open(fn) as f:
+        _defaults = yaml.safe_load(f)
+
+    update_defaults(_defaults)
+    del fn, _defaults
diff --git a/dask/dask.yaml b/dask/dask.yaml
new file mode 100644
index 000000000..1b6a2a27e
--- /dev/null
+++ b/dask/dask.yaml
@@ -0,0 +1,1 @@
+temporary-directory: null     # Directory for local disk like /tmp, /scratch, or /local


Test patch:
diff --git a/dask/tests/test_config.py b/dask/tests/test_config.py
index 4c4ecf5f2..1f6f4f891 100644
--- a/dask/tests/test_config.py
+++ b/dask/tests/test_config.py
@@ -392,3 +392,7 @@ def test_get_set_roundtrip(key):
 
 def test_merge_None_to_dict():
     assert dask.config.merge({'a': None, 'c': 0}, {'a': {'b': 1}}) == {'a': {'b': 1}, 'c': 0}
+
+
+def test_core_file():
+    assert 'temporary-directory' in dask.config.config
Repository: dask/dask
Pull request #5384 (2019-09-09 23:32:21)

Support numbers and None in parse_bytes/timedelta
These changes were downstream in distributed, and are useful when dealing with older configuration files.


Patch:
diff --git a/dask/utils.py b/dask/utils.py
index 6bc9e1f33..a89ba6ee0 100644
--- a/dask/utils.py
+++ b/dask/utils.py
@@ -1158,9 +1158,13 @@ def parse_bytes(s):
     1000000000
     >>> parse_bytes('MB')
     1000000
+    >>> parse_bytes(123)
+    123
     >>> parse_bytes('5 foos')  # doctest: +SKIP
     ValueError: Could not interpret 'foos' as a byte unit
     """
+    if isinstance(s, (int, float)):
+        return int(s)
     s = s.replace(" ", "")
     if not s[0].isdigit():
         s = "1" + s
@@ -1292,6 +1296,8 @@ def parse_timedelta(s, default="seconds"):
     >>> parse_timedelta(timedelta(seconds=3))  # also supports timedeltas
     3
     """
+    if s is None:
+        return None
     if isinstance(s, timedelta):
         s = s.total_seconds()
         return int(s) if int(s) == s else s


Test patch:
diff --git a/dask/tests/test_utils.py b/dask/tests/test_utils.py
index 3b9aece3c..fce337ed4 100644
--- a/dask/tests/test_utils.py
+++ b/dask/tests/test_utils.py
@@ -481,6 +481,7 @@ def test_parse_bytes():
     assert parse_bytes("1e6") == 1000000
     assert parse_bytes("1e6 kB") == 1000000000
     assert parse_bytes("MB") == 1000000
+    assert parse_bytes(123) == 123
 
 
 def test_parse_timedelta():
@@ -497,11 +498,13 @@ def test_parse_timedelta():
         ("1 ns", 1e-9),
         ("2m", 120),
         ("2 minutes", 120),
+        (None, None),
+        (3, 3),
         (datetime.timedelta(seconds=2), 2),
         (datetime.timedelta(milliseconds=100), 0.1),
     ]:
         result = parse_timedelta(text)
-        assert abs(result - value) < 1e-14
+        assert result == value or abs(result - value) < 1e-14
 
     assert parse_timedelta("1ms", default="seconds") == 0.001
     assert parse_timedelta("1", default="seconds") == 1
Repository: dask/dask
Pull request #5396 (2019-09-11 21:33:41)

Add support for numpy.vectorize in funcname
This PR adds support for `numpy.vectorize` objects to `funcname` in `utils.py` (similar to `toolz.curry` and `multipledispatch`). `numpy.vectorize` objects don't have a `__name__` attribute so currently `funcname` returns the string representation of the object:

```python
In [1]: import numpy as np

In [2]: from dask.utils import funcname

In [3]: funcname(np.vectorize(int))
Out[3]: '<numpy.vectorize object at 0x110280400>'
```

This impacts `da.gufunc` in that when `vectorize=True` a deterministic `.name` is not produced. For example, running:

```python
import numpy as np
import dask.array as da

a = np.arange(25, dtype=np.float64).reshape(5, 5)
x = da.from_array(a, chunks=(2, 2))

print('With vectorize=True')
for _ in range(3):
    # These will all be different
    print(da.gufunc(int, signature="()->()", output_dtypes="int", vectorize=True)(x).name)
    
print('With vectorize=False')
for _ in range(3):
    # These will all be the same
    print(da.gufunc(int, signature="()->()", output_dtypes="int", vectorize=False)(x).name)
```

outputs:

```
With vectorize=True
transpose-8ea88bc4583435ddb2df090b1ec7656f
transpose-c69ba9a64d19bbc94d9e215a8240c338
transpose-223766214d0c94c74a8f036099a60b79
With vectorize=False
transpose-4eedbcddb1b68531b9a52ba6b6219f07
transpose-4eedbcddb1b68531b9a52ba6b6219f07
transpose-4eedbcddb1b68531b9a52ba6b6219f07
```


- [ ] Tests added / passed
- [ ] Passes `black dask` / `flake8 dask`


jrbourbeau: An alternative fix would be to add a `__name__` attribute to `numpy.vectorize` objects upstream in NumPy. Perhaps @shoyer has thoughts on whether that's appropriate? 
mrocklin: +1
shoyer: > An alternative fix would be to add a `__name__` attribute to `numpy.vectorize` objects upstream in NumPy. Perhaps @shoyer has thoughts on whether that's appropriate?

Sure, that seems reasonable.


Patch:
diff --git a/dask/utils.py b/dask/utils.py
index a89ba6ee0..ecb7113cf 100644
--- a/dask/utils.py
+++ b/dask/utils.py
@@ -686,6 +686,9 @@ def funcname(func):
     # multipledispatch objects
     if "multipledispatch" in module_name and "Dispatcher" == type_name:
         return func.name
+    # numpy.vectorize objects
+    if "numpy" in module_name and "vectorize" == type_name:
+        return "vectorize_" + func.pyfunc.__name__
 
     # All other callables
     try:


Test patch:
diff --git a/dask/tests/test_utils.py b/dask/tests/test_utils.py
index fce337ed4..d99c89c18 100644
--- a/dask/tests/test_utils.py
+++ b/dask/tests/test_utils.py
@@ -346,6 +346,14 @@ def test_funcname_multipledispatch():
     assert funcname(functools.partial(foo, a=1)) == "foo"
 
 
+def test_funcname_numpy_vectorize():
+    np = pytest.importorskip("numpy")
+
+    func = np.vectorize(int)
+
+    assert funcname(func) == "vectorize_int"
+
+
 def test_ndeepmap():
     L = 1
     assert ndeepmap(0, inc, L) == 2
Repository: dask/dask
Pull request #5420 (2019-09-20 17:14:07)

Use sentinel no_default in get_dependencies
This PR updates our handling of the default value for `task=` in `get_dependencies` to use a sentinel `no_default` value instead of `None` (this helps with `is not None` checks in `get_dependencies`). Fixes https://github.com/dask/distributed/issues/2756

- [ ] Tests added / passed
- [ ] Passes `black dask` / `flake8 dask`


jrbourbeau: I guess `None` is also a valid key...perhaps we should use `no_default` there too

Patch:
diff --git a/dask/core.py b/dask/core.py
index b7f2d02f2..ec384848d 100644
--- a/dask/core.py
+++ b/dask/core.py
@@ -2,6 +2,8 @@ from itertools import chain
 
 from .utils_test import add, inc  # noqa: F401
 
+no_default = "__no_default__"
+
 
 def ishashable(x):
     """ Is x hashable?
@@ -152,7 +154,7 @@ def get(dsk, out, cache=None):
     return result
 
 
-def get_dependencies(dsk, key=None, task=None, as_list=False):
+def get_dependencies(dsk, key=None, task=no_default, as_list=False):
     """ Get the immediate tasks on which this task depends
 
     Examples
@@ -183,7 +185,7 @@ def get_dependencies(dsk, key=None, task=None, as_list=False):
     """
     if key is not None:
         arg = dsk[key]
-    elif task is not None:
+    elif task is not no_default:
         arg = task
     else:
         raise ValueError("Provide either key or task")


Test patch:
diff --git a/dask/tests/test_core.py b/dask/tests/test_core.py
index df1e1fb84..95e513586 100644
--- a/dask/tests/test_core.py
+++ b/dask/tests/test_core.py
@@ -133,6 +133,12 @@ def test_get_dependencies_many():
     assert s == []
 
 
+def test_get_dependencies_task_none():
+    # Regression test for https://github.com/dask/distributed/issues/2756
+    dsk = {"foo": None}
+    assert get_dependencies(dsk, task=dsk["foo"]) == set()
+
+
 def test_get_deps():
     """
     >>> dsk = {'a': 1, 'b': (inc, 'a'), 'c': (inc, 'b')}