nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: intake/intake
Pull request #530 (2020-09-16 17:18:39)

issue 529 fix with test
I had to modify one of the existing test cases and added a new one with a lone param. The test_utils passes however I am not sure what global effect this change could have. 
@martindurant please review


Patch:
diff --git a/intake/source/utils.py b/intake/source/utils.py
index 254f401..1217031 100644
--- a/intake/source/utils.py
+++ b/intake/source/utils.py
@@ -245,16 +245,17 @@ def path_to_glob(path):
     if not isinstance(path, str):
         return path
 
-    # Get just the real bits of the urlpath
-    literal_texts = [i[0] for i in fmt.parse(path)]
-
-    # Only use a star for first empty string in literal_texts
-    index_of_empty = [i for i, lt in enumerate(literal_texts) if lt == '' and i != 0]
-    glob = '*'.join([literal_texts[i] for i in range(len(literal_texts)) if i not in index_of_empty])
-
+    # calculate glob expression
+    glob = ''
+    prev_field_name = None
+    for literal_text, field_name, _, _ in fmt.parse(path):
+        glob += literal_text
+        # condition to avoid repeated * on adjacent fields
+        if field_name and (literal_text or prev_field_name is None):
+            glob += '*'
+        prev_field_name = field_name
     return glob
 
-
 def path_to_pattern(path, metadata=None):
     """
     Remove source information from path when using chaching


Test patch:
diff --git a/intake/source/tests/test_utils.py b/intake/source/tests/test_utils.py
index 8c872b8..c644fd8 100644
--- a/intake/source/tests/test_utils.py
+++ b/intake/source/tests/test_utils.py
@@ -19,6 +19,7 @@ from intake.source.utils import (
     ('{year}/{month}/{day}.csv', '*/*/*.csv'),
     ('data/**/*.csv', 'data/**/*.csv'),
     ('data/{year:4}{month:02}{day:02}.csv', 'data/*.csv'),
+    ('{lone_param}','*')
 ])
 def test_path_to_glob(pattern, expected):
     assert path_to_glob(pattern) == expected
Repository: intake/intake
Pull request #566 (2021-01-21 21:51:04)

add shape and dtype properties to ZarrArraySource
`ZarrArraySource` now has `shape` and `dtype` properties which default to `None` and are set to the respective properties of the underlying `_arr` after creating the dask array wrapping the zarr array. The one test for `ZarrArraySource` now checks for these properties.



Patch:
diff --git a/intake/source/zarr.py b/intake/source/zarr.py
index 60d9a2c..686088b 100644
--- a/intake/source/zarr.py
+++ b/intake/source/zarr.py
@@ -43,6 +43,7 @@ class ZarrArraySource(DataSource):
         self.storage_options = storage_options or {}
         self.component = component
         self.kwargs = kwargs
+        self.chunks = None
         self._arr = None
         super(ZarrArraySource, self).__init__(metadata=metadata)
 
@@ -53,6 +54,8 @@ class ZarrArraySource(DataSource):
                                      storage_options=self.storage_options,
                                      **self.kwargs)
             self.chunks = self._arr.chunks
+            self.shape = self._arr.shape
+            self.dtype = self._arr.dtype
             self.npartitions = self._arr.npartitions
         return Schema(dtype=str(self.dtype), shape=self.shape,
                       extra_metadata=self.metadata,


Test patch:
diff --git a/intake/source/tests/test_npy.py b/intake/source/tests/test_npy.py
index ff87d45..a5528e0 100644
--- a/intake/source/tests/test_npy.py
+++ b/intake/source/tests/test_npy.py
@@ -66,6 +66,8 @@ def test_zarr_minimal():
     cat = intake.open_catalog(posixpath.join(here, 'sources.yaml'))
     s = cat.zarr1()
     assert s.container == 'ndarray'
-    assert s.read().tolist() == [73, 98, 46, 38, 20, 12, 31,  8, 89, 72]
+    assert s.read().tolist() == [73, 98, 46, 38, 20, 12, 31, 8, 89, 72]
     assert s.npartitions == 1
+    assert s.dtype == 'int'
+    assert s.shape == (10,)
     assert (s.read_partition((0, )) == s.read()).all()
Repository: intake/intake
Pull request #588 (2021-04-02 15:18:04)

Fix nested catalog access
I noticed in __getitem__ that a tuple-style method of accessing nested entries is supposed to work, but it does not.

For example:
`catalog.root.leaf` works fine, but `catalog["root", "leaf"]` fails with a KeyError. This PR is meant to cause the latter behavior to work as stated in the __getitem__ docstring.


Patch:
diff --git a/intake/catalog/base.py b/intake/catalog/base.py
index d98f517..3aabadf 100644
--- a/intake/catalog/base.py
+++ b/intake/catalog/base.py
@@ -412,7 +412,7 @@ class Catalog(DataSource):
         elif isinstance(key, tuple):
             out = self
             for part in key:
-                out = self[part]
+                out = out[part]
             return out()
         raise KeyError(key)
 


Test patch:
diff --git a/intake/tests/catalog_nested.yml b/intake/tests/catalog_nested.yml
new file mode 100644
index 0000000..cd04fc0
--- /dev/null
+++ b/intake/tests/catalog_nested.yml
@@ -0,0 +1,6 @@
+sources:
+  nested:
+    description: References catalog_nested_sub.yml
+    driver: yaml_file_cat
+    args:
+      path: "{{ CATALOG_DIR }}__unit_test_catalog_nested_sub.yml"
diff --git a/intake/tests/catalog_nested_sub.yml b/intake/tests/catalog_nested_sub.yml
new file mode 100644
index 0000000..9759ee3
--- /dev/null
+++ b/intake/tests/catalog_nested_sub.yml
@@ -0,0 +1,6 @@
+sources:
+  ex1:
+    description: this is a sub-resource
+    driver: csv
+    args:
+      urlpath: ""
diff --git a/intake/tests/test_top_level.py b/intake/tests/test_top_level.py
index aca5bfc..33b9a25 100644
--- a/intake/tests/test_top_level.py
+++ b/intake/tests/test_top_level.py
@@ -134,3 +134,21 @@ def test_no_imports():
     for mod in ['intake.tests', 'intake.interface', 'intake.source.csv',
                 'intake.cli', 'intake.auth']:
         assert mod not in sys.modules
+
+
+@pytest.fixture
+def tmp_path_catalog_nested():
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        tmp_path = posixpath.join(tmp_dir, 'intake')
+        target_catalog = copy_test_file('catalog_nested.yml', tmp_path)
+        copy_test_file('catalog_nested_sub.yml', tmp_path)
+        yield target_catalog
+
+
+def test_nested_catalog_access(tmp_path_catalog_nested):
+    cat = intake.open_catalog(tmp_path_catalog_nested)
+    entry1 = cat.nested.ex1
+    entry2 = cat["nested.ex1"]
+    entry3 = cat[["nested", "ex1"]]
+    entry4 = cat["nested", "ex1"]
+    assert entry1 == entry2 == entry3 == entry4
Repository: intake/intake
Pull request #597 (2021-05-19 01:13:52)

Support entrypoint style in import_name


Patch:
diff --git a/intake/source/__init__.py b/intake/source/__init__.py
index da23227..62a61c4 100644
--- a/intake/source/__init__.py
+++ b/intake/source/__init__.py
@@ -85,9 +85,18 @@ classes = {}
 
 def import_name(name):
     import importlib
-    mod, cls = name.rsplit('.', 1)
-    module = importlib.import_module(mod)
-    return getattr(module, cls)
+    if ":" in name:
+        if name.count(":") > 1:
+            raise ValueError("Cannot decipher name to import: %s", name)
+        mod, rest = name.split(":")
+        bit = importlib.import_module(mod)
+        for part in rest.split("."):
+            bit = getattr(bit, part)
+        return bit
+    else:
+        mod, cls = name.rsplit('.', 1)
+        module = importlib.import_module(mod)
+        return getattr(module, cls)
 
 
 def get_plugin_class(name):


Test patch:
diff --git a/intake/source/tests/test_base.py b/intake/source/tests/test_base.py
index 6f17418..32f4a77 100644
--- a/intake/source/tests/test_base.py
+++ b/intake/source/tests/test_base.py
@@ -14,6 +14,7 @@ import pandas as pd
 
 import intake.source.base as base
 import intake.source.derived as der
+import intake.source
 
 
 def test_datasource_base_method_exceptions():
@@ -315,3 +316,15 @@ def test_alias_fail():
     s.container == 'other'
     with pytest.raises(ValueError):
         s.read()
+
+
+@pytest.mark.parametrize("data", [
+    ("intake.source.import_name", intake.source.import_name),
+    ("intake.source:import_name", intake.source.import_name),
+    ("intake.source.DataSource", intake.source.DataSource),
+    ("intake.source:DataSource", intake.source.DataSource),
+    ("intake.source:base.DataSource", intake.source.DataSource),
+])
+def test_import_name(data):
+    text, object = data
+    assert intake.source.import_name(text) == object
Repository: intake/intake
Pull request #721 (2023-03-10 19:01:57)

Handle parsing UserParameters into non-string types


Patch:
diff --git a/intake/catalog/utils.py b/intake/catalog/utils.py
index e329d77..bc6ac8a 100644
--- a/intake/catalog/utils.py
+++ b/intake/catalog/utils.py
@@ -268,6 +268,18 @@ def coerce_datetime(v=None):
     return pandas.to_datetime(v)
 
 
+def with_str_parse(value, rule):
+    import ast
+
+    if isinstance(value, str) and rule not in [str, coerce_datetime]:
+        try:
+            value = ast.literal_eval(value)
+        except (ValueError, TypeError, RuntimeError):
+            pass
+
+    return rule(value)
+
+
 COERCION_RULES = {
     "bool": bool,
     "datetime": coerce_datetime,
@@ -312,7 +324,7 @@ def coerce(dtype, value):
                 raise ValueError("Failed to coerce string to list") from e
         return value
     op = COERCION_RULES[dtype]
-    out = op() if value is None else op(value)
+    out = op() if value is None else with_str_parse(value, op)
     if isinstance(out, dict) and inner is not None:
         # TODO: recurse into coerce here, to allow list[list[str]] and such?
         out = {k: COERCION_RULES[inner](v) for k, v in out.items()}


Test patch:
diff --git a/intake/catalog/tests/test_utils.py b/intake/catalog/tests/test_utils.py
index f169a0a..e040481 100644
--- a/intake/catalog/tests/test_utils.py
+++ b/intake/catalog/tests/test_utils.py
@@ -48,7 +48,22 @@ def test_flatten():
 
 @pytest.mark.parametrize(
     "value,dtype,expected",
-    [(1, "int", 1), ("1", "int", 1), (1, "str", "1"), ((), "list", []), ((1,), "list", [1]), ((1,), "list[str]", ["1"])],
+    [
+        (1, "int", 1),
+        ("1", "int", 1),
+        (1, "str", "1"),
+        ((), "list", []),
+        ((1,), "list", [1]),
+        ((1,), "list[str]", ["1"]),
+        ("{'a': 1}", "dict", {"a": 1}),
+        ("{'a': '1'}", "dict", {"a": "1"}),
+        ("[1, 2]", "list", [1, 2]),
+        ("['1', '2']", "list", ["1", "2"]),
+        ("[{'a': '1'}]", "list", [{"a": "1"}]),
+        ("{'a': [1, 2]}", "dict", {"a": [1, 2]}),
+        ("{'a': ['1', '2']}", "dict", {"a": ["1", "2"]}),
+        ("1988-02-24T13:37+0100", "datetime", pd.Timestamp("1988-02-24 13:37+0100")),
+    ],
 )
 def test_coerce(value, dtype, expected):
     out = utils.coerce(dtype, value)
Repository: intake/intake
Pull request #849 (2024-09-24 17:24:46)

Make cat.delete equivalent to del


Patch:
diff --git a/intake/readers/entry.py b/intake/readers/entry.py
index 220f4f1..a3c77c2 100644
--- a/intake/readers/entry.py
+++ b/intake/readers/entry.py
@@ -289,10 +289,9 @@ class Catalog(Tokenizable):
             Also removed data/entries references by the given one, and those
             they refer to in turn.
         """
-        item = self.get_entity(name)
-        del self[item]
         if recursive:
             raise NotImplementedError
+        del self[name]
 
     def extract_parameter(
         self,


Test patch:
diff --git a/intake/readers/tests/test_workflows.py b/intake/readers/tests/test_workflows.py
index f59f6c3..f0821d7 100644
--- a/intake/readers/tests/test_workflows.py
+++ b/intake/readers/tests/test_workflows.py
@@ -36,6 +36,11 @@ def test_pipelines_in_catalogs(dataframe_file, df):
     assert reader.equals(other=reader).read() is True
     assert cat.eq.read() is True
 
+    with pytest.raises(NotImplementedError):
+        cat.delete("eq", recursive=True)
+    cat.delete("eq")
+    assert "eq" not in cat
+
 
 def test_pipeline_steps(dataframe_file, df):
     data = intake.readers.datatypes.CSV(url=dataframe_file)
Repository: intake/intake
Pull request #858 (2025-01-09 15:10:44)

New UP type: named options
This is a useful multi-value UP type; but it does not solve the case of "I want the reader to choose between multiple possible data inputs", which probably requires changing the order in which arguments get processed.


Patch:
diff --git a/intake/readers/user_parameters.py b/intake/readers/user_parameters.py
index 31a2df1..9762892 100644
--- a/intake/readers/user_parameters.py
+++ b/intake/readers/user_parameters.py
@@ -108,8 +108,32 @@ class OptionsUserParameter(SimpleUserParameter):
         return self.coerce(value) in self.options
 
 
+class NamedOptionsUserParameter(SimpleUserParameter):
+    """One choice out of a given allow dictionary accessed by its keys
+
+    In this case, `dtype` is the types of the dictionary values, and
+    we have a separate "keytype" for the keys, str by default
+    """
+
+    def __init__(self, options, default, dtype=object, keytype=str, **kw):
+        super().__init__(default=options[default], dtype=dtype, **kw)
+        self.keytype = keytype
+        self.options = options
+
+    def coerce(self, value):
+        return self.options.get(value, None)
+
+    def _validate(self, value):
+        if not isinstance(value, self.keytype):
+            value = self.keytype(value)
+        return value in self.options.values()
+
+
 class MultiOptionUserParameter(OptionsUserParameter):
-    """Multiple choices out of a given allow list/tuple"""
+    """Multiple choices out of a given allow list/tuple
+
+    In this case, dtype is the type of each member of the list
+    """
 
     def __init__(self, options: list | tuple, dtype=object, **kw):
         super().__init__(options=options, dtype=dtype, **kw)


Test patch:
diff --git a/intake/readers/tests/test_up.py b/intake/readers/tests/test_up.py
index 97a0df5..a566fa7 100644
--- a/intake/readers/tests/test_up.py
+++ b/intake/readers/tests/test_up.py
@@ -22,3 +22,16 @@ def test_basic():
         # supplied None as a value to int parameter
         pars = {"k": ["{p}", 1], "p": None}
         up.set_values({"p": p}, pars)
+
+
+def test_named_options():
+    from intake.readers import user_parameters as up
+
+    p = up.NamedOptionsUserParameter({"a": "athing", "b": "bthing"}, default="b")
+    pars = {"k": ["{p}", 1]}
+    out = up.set_values({"p": p}, pars)
+    assert out == {"k": ["bthing", 1]}
+
+    pars = {"k": ["{p}", 1], "p": "a"}
+    out = up.set_values({"p": p}, pars)
+    assert out == {"k": ["athing", 1]}