Repository: datalad/datalad-next

Issue:
Allow for URL handler definition in config
Presently `AnyUrlOperations` only consults a runtime/code registry. We could...

Specify URL handlers in configuration:

The items would be of syntax: `datalad.url-handler.<regex>.<property>`,
where `<regex>` is a match expression when to apply a particular handler.

Putting the regex into the section name would prevent the problem of undesired shadowing, e.g. when some software package installation defines `datalad.url-handler.openneuro` and I already had that, but for a different match expression. Ultimately the match expressions are the unique identifier and the git config syntax allows for putting them into the subsection names.

There can be any number of different `<property>`.

The special property `class` is something like `package.module.class` that identifies what class to import and use as a handler. Any other property would be passed on to the handlers constructor as `kwargs` (after applying `.replace('-', '_')` to the property name).

Alternatively (or maybe in addition), we could have a `kwargs` property to provide JSON-encoded arguments.

Or we could try JSON-decode any property-value, and only fall back on the
str-representation if that does not work.

Both of the latter approaches would make it easier to specify non-string
values. I am thinking of cases where it would be useful to provide
timeout durations, number of retries, boolean flags, etc.

Update: real-world arguments that need to be specified can become rather complex quickly. Example:

```
{'fs_kwargs': {'anon': True, 's3': {'anon': True}}}
```

anything other than a single `kwargs` property with a JSON encodes value seems to have little applicability.


Discussion:

Patch:
diff --git a/datalad_next/url_operations/any.py b/datalad_next/url_operations/any.py
index 7ab94ad..c91d94a 100644
--- a/datalad_next/url_operations/any.py
+++ b/datalad_next/url_operations/any.py
@@ -4,18 +4,14 @@
 from __future__ import annotations
 
 from importlib import import_module
+import json
 import logging
 from pathlib import Path
 import re
 from typing import Dict
-from urllib import (
-    request,
-    parse,
-)
-
-from .http import HttpUrlOperations
-from .file import FileUrlOperations
-from .ssh import SshUrlOperations
+
+from datalad_next.exceptions import CapturedException
+
 from . import UrlOperations
 
 lgr = logging.getLogger('datalad.ext.next.url_operations.any')
@@ -24,16 +20,28 @@ lgr = logging.getLogger('datalad.ext.next.url_operations.any')
 __all__ = ['AnyUrlOperations']
 
 # define handlers for each supported URL pattern
-# the key in this dict is a regex match expression.
-# the value is a tuple of containing module, and name of the
-# class providing the handler
-# extensions could patch their's in
+# FORMAT OF HANDLER REGISTRY (dict)
+# - key: regex match expression to be apply on a URL (to test whether a
+#   particular handler should be used for a given URL)
+# - value: tuple (handler specification, see below)
+# FORMAT OF HANDLER SPECIFICATION
+# - tuple of min-length 1
+# - item1: str, handler class to import
+#   e.g., package.module.class
+# - item2: dict, optional, kwargs to pass to the handler constructor
+
 # TODO support proper entrypoint mechanism
-_url_handlers = dict(
-    http=('datalad_next.url_operations.http', 'HttpUrlOperations'),
-    file=('datalad_next.url_operations.file', 'FileUrlOperations'),
-    ssh=('datalad_next.url_operations.ssh', 'SshUrlOperations'),
-)
+# It is best to only record handlers here for which there is no alternative,
+# because the best handler is determined based on this information
+# and only this handler is imported. If that fails, there is no fallback.
+# Handlers that may or may not work under given conditions should only
+# be added via external logic after they have been found to be "working"
+# on a given installation.
+_url_handlers = {
+    'http': ('datalad_next.url_operations.http.HttpUrlOperations',),
+    'file': ('datalad_next.url_operations.file.FileUrlOperations',),
+    'ssh': ('datalad_next.url_operations.ssh.SshUrlOperations',),
+}
 
 
 class AnyUrlOperations(UrlOperations):
@@ -52,13 +60,64 @@ class AnyUrlOperations(UrlOperations):
     efficiently.
     """
     def __init__(self, cfg=None):
+        """
+        Parameters
+        ----------
+        cfg: ConfigManager, optional
+          A config manager instance that is consulted for any configuration
+          filesystem configuration individual handlers may support.
+        """
         super().__init__(cfg=cfg)
-        self._url_handlers = {
-            re.compile(k): v for k, v in _url_handlers.items()
-        }
+        self._load_handler_registery()
         # cache of already used handlers
         self._url_handler_cache = dict()
 
+    def _load_handler_registery(self):
+        # update with handlers from config
+        # https://github.com/datalad/datalad-next/issues/217
+        cfgh = {}
+        for citem in self.cfg.keys():
+            if not citem.startswith('datalad.url-handler.'):
+                # none of our business
+                continue
+            # the match expression is right in the item key
+            # (all but the first two and the last segment)
+            citem_l = citem.split('.')
+            match = '.'.join(citem_l[2:-1])
+            prop = citem_l[-1]
+            value = self.cfg[citem]
+            if prop != 'class':
+                try:
+                    value = json.loads(value)
+                except Exception as e:
+                    ce = CapturedException(e)
+                    lgr.debug(
+                        'Ignoring invalid URL handler configuration '
+                        'for %r(%s): %r [%s]',
+                        match, prop, value, ce)
+                    continue
+            hc = cfgh.get(match, {})
+            hc[prop] = value
+            cfgh[match] = hc
+        # merge all specs
+        uh = dict(_url_handlers)
+        for match, spec in cfgh.items():
+            try:
+                uh[match] = (spec['class'], spec['kwargs'])
+            except KeyError:
+                try:
+                    uh[match] = (spec['class'],)
+                except Exception as e:
+                    CapturedException(e)
+                    lgr.debug(
+                        'Ignoring incomplete URL handler specification '
+                        'for %r: %r', match, spec)
+        self._url_handlers = {}
+        for k, v in uh.items():
+            # compile matches to finalize
+            lgr.log(8, 'Add URL handler for %r: %r', k, v)
+            self._url_handlers[re.compile(k)] = v
+
     def _get_handler(self, url: str) -> UrlOperations:
         # match URL against all registered handlers and get the one with the
         # longest (AKA best) match
@@ -83,10 +142,17 @@ class AnyUrlOperations(UrlOperations):
 
         # we need to import the handler
         try:
-            mod, cls = self._url_handlers[best_match]
+            handler_spec = self._url_handlers[best_match]
+            # split the import declaration into units
+            toimport = handler_spec[0].split('.')
+            # the handler class is the last unit
+            cls = toimport[-1]
+            # the rest is the module
+            mod = '.'.join(toimport[:-1])
             module = import_module(mod, package='datalad')
             handler_cls = getattr(module, cls)
-            url_handler = handler_cls(cfg=self.cfg)
+            handler_kwargs = handler_spec[1] if len(handler_spec) > 1 else {}
+            url_handler = handler_cls(cfg=self.cfg, **handler_kwargs)
         except Exception as e:
             raise ValueError(
                 'Cannot create URL handler instance for '


Test patch:
diff --git a/datalad_next/url_operations/tests/test_any.py b/datalad_next/url_operations/tests/test_any.py
index 1ac5705..e7be8bb 100644
--- a/datalad_next/url_operations/tests/test_any.py
+++ b/datalad_next/url_operations/tests/test_any.py
@@ -5,9 +5,9 @@ from .. import (
 from ..any import (
     _url_handlers,
     AnyUrlOperations,
-    HttpUrlOperations,
-    FileUrlOperations,
 )
+from ..http import HttpUrlOperations
+from ..file import FileUrlOperations
 
 
 def test_get_best_url_handler(monkeypatch):
@@ -19,7 +19,7 @@ def test_get_best_url_handler(monkeypatch):
         m.setitem(
             _url_handlers,
             'https://ex.*\.co',
-            ('datalad_next.url_operations.file', 'FileUrlOperations'),
+            ('datalad_next.url_operations.file.FileUrlOperations',),
         )
         # the handlers are sucked into the class, so we need a new instance
         ops = AnyUrlOperations()