Repository: datalad/datalad

Issue:
clone/install: introduce url mapping convenience?
#### What is the problem?

ATM for a user visiting some datalad-osf published dataset on osf.io it would require either user knowledge or every dataset having a description such as "this component can be git or datalad cloned from a 'osf://ID' URL, where 'ID' is the OSF node ID that shown in the OSF HTTP URL, e.g. https://osf.io/q8xnk/ can be cloned from osf://q8xnk", and then do all url tune up manually to get `datalad` `clone` or `install` working.

### Suggested solution:

I think datalad should make it easier and allow for a simple "copy paste URL I see in the browser bar", thus making `datalad clone https://osf.io/q8xnk/` work as well, while internally mapping it into `osf://q8xnk` for `git clone` operation.

Possible cons:
- if OSF changes their url schema -- would break our mapping.  But given historical stability and presence of those URLs in the wild already, it is very unlikely to happen

Possible implementation:
- datalad internally and also allowing extensions to register new "URL mappers"
- such "mappers" possibly should allow for different/absent context/purpose behavior
  - e.g. for OSF it would only matter for `clone`
  - in a somewhat related (waiting for merge) #4816, where within datalad itself we introduce "mapping" of `shub://` URLs into (minted by singularity hub with timeouts) http urls so that those images could be downloaded from the singularity hub; context would be `download`
- we already hardcode `///` to be a mapping to `http://datasets.datalad.org` -- implementation could generalize it
- some mappers could be as simple as regex based transformation (like in the case of OSF, `///`)

Comments:

Patch:
diff --git a/datalad/core/distributed/clone.py b/datalad/core/distributed/clone.py
index dfed3f13f..6f1ff6778 100644
--- a/datalad/core/distributed/clone.py
+++ b/datalad/core/distributed/clone.py
@@ -11,7 +11,6 @@
 
 import logging
 import re
-import requests
 from os.path import expanduser
 from collections import OrderedDict
 from urllib.parse import unquote as urlunquote
@@ -32,7 +31,6 @@ from datalad.cmd import (
     CommandError,
     GitWitlessRunner,
     StdOutCapture,
-    StdOutErrCapture,
 )
 from datalad.distributed.ora_remote import (
     LocalIO,
@@ -47,6 +45,7 @@ from datalad.support.constraints import (
 )
 from datalad.support.exceptions import DownloadError
 from datalad.support.param import Parameter
+from datalad.support.strings import get_replacement_dict
 from datalad.support.network import (
     get_local_file_url,
     download_url,
@@ -62,6 +61,7 @@ from datalad.dochelpers import (
 )
 from datalad.utils import (
     ensure_bool,
+    ensure_list,
     knows_annex,
     make_tempfile,
     Path,
@@ -122,6 +122,26 @@ class Clone(Interface):
     result), or found already installed from the specified source ('notneeded'
     result).<< PYTHON ||
 
+    URL mapping configuration
+
+    'clone' supports the transformation of URLs via (multi-part) substitution
+    specifications. A substitution specification is defined as a configuration
+    setting 'datalad.clone.url-substition.<seriesID>' with a string containing
+    a match and substitution expression, each following Python's regular
+    expression syntax. Both expressions are concatenated to a single string
+    with an arbitrary delimiter character. The delimiter is defined by
+    prefixing the string with the delimiter. Prefix and delimiter are stripped
+    from the expressions (Example: ",^http://(.*)$,https://\\1").  This setting
+    can be defined multiple times, using the same '<seriesID>'.  Substitutions
+    in a series will be applied incrementally, in order of their definition.
+    The first substitution in such a series must match, otherwise no further
+    substitutions in a series will be considered. However, following the first
+    match all further substitutions in a series are processed, regardless
+    whether intermediate expressions match or not. Substitution series themselves
+    have no particular order, each matching series will result in a candidate
+    clone URL. Consequently, the initial match specification in a series should
+    be as precise as possible to prevent inflation of candidate URLs.
+
     .. seealso::
 
       :ref:`handbook:3-001`
@@ -351,6 +371,74 @@ class Clone(Interface):
                 ds.subdatasets(path, set_property=[("datalad-url", source)])
 
 
+def _get_url_mappings(cfg):
+    cfg_prefix = 'datalad.clone.url-substitute.'
+    # figure out which keys we should be looking for
+    # in the active config
+    subst_keys = set(k for k in cfg.keys() if k.startswith(cfg_prefix))
+    # and in the common config specs
+    from datalad.interface.common_cfg import definitions
+    subst_keys.update(k for k in definitions if k.startswith(cfg_prefix))
+    # TODO a potential sorting of substitution series could be implemented
+    # here
+    return [
+        # decode the rule specifications
+        get_replacement_dict(
+            # one or more could come out
+            ensure_list(
+                cfg.get(
+                    k,
+                    # make sure to pull the default from the common config
+                    default=cfg.obtain(k),
+                    # we specifically support declaration of multiple
+                    # settings to build replacement chains
+                    get_all=True)))
+        for k in subst_keys
+    ]
+
+
+def _map_urls(cfg, urls):
+    mapping_specs = _get_url_mappings(cfg)
+    if not mapping_specs:
+        return urls
+
+    mapped = []
+    # we process the candidate in order to maintain any prioritization
+    # encoded in it (e.g. _get_flexible_source_candidates_for_submodule)
+    # if we have a matching mapping replace the URL in its position
+    for u in urls:
+        # we only permit a single match
+        # TODO we likely want to RF this to pick the longest match
+        mapping_applied = False
+        # try one mapping set at a time
+        for mapping_spec in mapping_specs:
+            # process all substitution patterns in the specification
+            # always operate on strings (could be a Path instance too)
+            mu = str(u)
+            matched = False
+            for match_ex, subst_ex in mapping_spec.items():
+                if not matched:
+                    matched = re.match(match_ex, mu) is not None
+                if not matched:
+                    break
+                # try to map, would return unchanged, if there is no match
+                mu = re.sub(match_ex, subst_ex, mu)
+            if mu != u:
+                lgr.debug("URL substitution: '%s' -> '%s'", u, mu)
+                mapped.append(mu)
+                # we could consider breaking after the for effective mapping
+                # specification. however, that would mean any generic
+                # definition of a broadly matching substitution would derail
+                # the entroe system. moreover, suddently order would matter
+                # substantially
+                mapping_applied = True
+        if not mapping_applied:
+            # none of the mappings matches, go with the original URL
+            # (really original, not the stringified one)
+            mapped.append(u)
+    return mapped
+
+
 def clone_dataset(
         srcs,
         destds,
@@ -410,6 +498,11 @@ def clone_dataset(
 
     dest_path = destds.pathobj
 
+    # check for configured URL mappings, either in the given config manager
+    # or in the one of the destination dataset, which is typically not existent
+    # yet and the process config manager is then used effectively
+    srcs = _map_urls(cfg or destds.config, srcs)
+
     # decode all source candidate specifications
     candidate_sources = [decode_source_spec(s, cfg=cfg) for s in srcs]
 
diff --git a/datalad/interface/common_cfg.py b/datalad/interface/common_cfg.py
index 319d6c0be..7485cf577 100644
--- a/datalad/interface/common_cfg.py
+++ b/datalad/interface/common_cfg.py
@@ -25,8 +25,53 @@ from datalad.utils import on_windows
 
 dirs = AppDirs("datalad", "datalad.org")
 
+subst_rule_docs = """\
+A substitution specification is a string with a match and substitution
+expression, each following Python's regular expression syntax. Both expressions
+are concatenated to a single string with an arbitrary delimiter character. The
+delimiter is defined by prefixing the string with the delimiter. Prefix and
+delimiter are stripped from the expressions (Example:
+",^http://(.*)$,https://\\1").  This setting can be defined multiple times.
+Substitutions will be applied incrementally, in order of their definition. The
+first substitution in such a series must match, otherwise no further
+substitutions in a series will be considered. However, following the first
+match all further substitutions in a series are processed, regardless whether
+intermediate expressions match or not."""
 
 definitions = {
+    'datalad.clone.url-substitute.github': {
+        'ui': ('question', {
+               'title': 'GitHub URL substitution rule',
+               'text': 'Mangling for GitHub-related URL. ' + subst_rule_docs
+        }),
+        'destination': 'global',
+        'default': (
+            # take any github project URL apart into <org>###<identifier>
+            r',https?://github.com/([^/]+)/(.*)$,\1###\2',
+            # replace any (back)slashes with a single dash
+            r',[/\\]+,-',
+            # replace any whitespace (include urlquoted variant)
+            # with a single underscore
+            r',\s+|(%2520)+|(%20)+,_',
+            # rebuild functional project URL
+            r',([^#]+)###(.*),https://github.com/\1/\2',
+        )
+    },
+    # TODO this one should migrate to the datalad-osf extension. however, right
+    # now extensions cannot provide default configuration
+    # https://github.com/datalad/datalad/issues/5769
+    'datalad.clone.url-substitute.osf': {
+        'ui': ('question', {
+               'title': 'Open Science Framework URL substitution rule',
+               'text': 'Mangling for OSF-related URLs. ' + subst_rule_docs
+        }),
+        'destination': 'global',
+        'default': (
+            # accept browser-provided URL and convert to those accepted by
+            # the datalad-osf extension
+            r',^https://osf.io/([^/]+)[/]*$,osf://\1',
+        )
+    },
     # this is actually used in downloaders, but kept cfg name original
     'datalad.crawl.cache': {
         'ui': ('yesno', {


Test patch:
diff --git a/datalad/core/distributed/tests/test_clone.py b/datalad/core/distributed/tests/test_clone.py
index 7c7bd2f55..30c30ebbf 100644
--- a/datalad/core/distributed/tests/test_clone.py
+++ b/datalad/core/distributed/tests/test_clone.py
@@ -1564,3 +1564,85 @@ def test_clone_recorded_subds_reset(path):
     eq_(ds_b.subdatasets()[0]["gitshasum"],
         sub_repo.get_hexsha(
             sub_repo.get_corresponding_branch(branch) or branch))
+
+
+@with_tempfile
+@with_tempfile
+def test_clone_url_mapping(src_path, dest_path):
+    src = create(src_path)
+    dest = Dataset(dest_path)
+    # check that the impossible doesn't work
+    assert_raises(IncompleteResultsError, clone, 'rambo', dest_path)
+    # rather than adding test URL mapping here, consider
+    # test_url_mapping_specs(), it is cheaper there
+  
+    # anticipate windows test paths and escape them
+    escaped_subst = (r',rambo,%s' % src_path).replace('\\', '\\\\')
+    for specs in (
+            # we can clone with a simple substitution
+            {'datalad.clone.url-substitute.mike': escaped_subst},
+            # a prior match to a dysfunctional URL doesn't impact success
+            {
+                'datalad.clone.url-substitute.no': ',rambo,picknick',
+                'datalad.clone.url-substitute.mike': escaped_subst,
+            }):
+        try:
+            with patch.dict(dest.config._merged_store, specs):
+                clone('rambo', dest_path)
+        finally:
+            dest.remove(check=False)
+
+    # check submodule config impact
+    dest.create()
+    with patch.dict(dest.config._merged_store,
+                    {'datalad.clone.url-substitute.mike': escaped_subst}):
+        dest.clone('rambo', 'subds')
+    submod_rec = dest.repo.get_submodules()[0]
+    # we record the original-original URL
+    eq_(submod_rec['gitmodule_datalad-url'], 'rambo')
+    # and put the effective one as the primary URL
+    eq_(submod_rec['gitmodule_url'], src_path)
+
+
+_nomatch_map = {
+    'datalad.clone.url-substitute.nomatch': (
+        ',nomatch,NULL',
+    )
+}
+_windows_map = {
+    'datalad.clone.url-substitute.win': (
+        r',C:\\Users\\datalad\\from,D:\\to',
+    )
+}
+
+
+def test_url_mapping_specs():
+    from datalad.core.distributed.clone import _map_urls
+    cfg = ConfigManager()
+    for m, i, o in (
+            # path redirect on windows
+            (_windows_map,
+             r'C:\Users\datalad\from',
+             r'D:\to'),
+            # test standard github mapping, no pathc needed
+            ({},
+             'https://github.com/datalad/testrepo_gh/sub _1',
+             'https://github.com/datalad/testrepo_gh-sub__1'),
+            # and on deep subdataset too
+            ({},
+             'https://github.com/datalad/testrepo_gh/sub _1/d/sub_-  1',
+             'https://github.com/datalad/testrepo_gh-sub__1-d-sub_-_1'),
+            # test that the presence of another mapping spec doesn't ruin
+            # the outcome
+            (_nomatch_map,
+             'https://github.com/datalad/testrepo_gh/sub _1',
+             'https://github.com/datalad/testrepo_gh-sub__1'),
+            # verify OSF mapping, but see
+            # https://github.com/datalad/datalad/issues/5769 for future
+            # implications
+            ({},
+             'https://osf.io/q8xnk/',
+             'osf://q8xnk'),
+            ):
+        with patch.dict(cfg._merged_store, m):
+            eq_(_map_urls(cfg, [i]), [o])
diff --git a/datalad/distribution/tests/test_install.py b/datalad/distribution/tests/test_install.py
index a86addfb8..769682e2a 100644
--- a/datalad/distribution/tests/test_install.py
+++ b/datalad/distribution/tests/test_install.py
@@ -919,3 +919,41 @@ def test_relpath_semantics(path):
         sub = install(
             dataset='super', source='subsrc', path=op.join('super', 'sub'))
         eq_(sub.path, op.join(super.path, 'sub'))
+
+
+def _create_test_install_recursive_github(path):  # pragma: no cover
+    # to be ran once to populate a hierarchy of test datasets on github
+    # Making it a full round-trip would require github credentials on CI etc
+    ds = create(opj(path, "testrepo  gh"))
+    # making them with spaces and - to ensure that we consistently use the mapping
+    # for create and for get/clone/install
+    ds.create("sub _1")
+    ds.create("sub _1/d/sub_-  1")
+    import datalad.distribution.create_sibling_github  # to bind API
+    ds.create_sibling_github(
+        "testrepo  gh",
+        github_organization='datalad',
+        recursive=True,
+        # yarik forgot to push first, "replace" is not working in non-interactive IIRC
+        # existing='reconfigure'
+    )
+    return ds.push(recursive=True, to='github')
+
+
+@skip_if_no_network
+@with_tempfile(mkdir=True)
+def test_install_recursive_github(path):
+    # test recursive installation of a hierarchy of datasets created on github
+    # using datalad create-sibling-github.  Following invocation was used to poplate it
+    #
+    # out = _create_test_install_recursive_github(path)
+
+    # "testrepo  gh" was mapped by our sanitization in create_sibling_github to testrepo_gh, thus
+    for i, url in enumerate([
+        'https://github.com/datalad/testrepo_gh',
+        # optionally made available to please paranoids, but with all takes too long (22sec)
+        #'https://github.com/datalad/testrepo_gh.git',
+        #'git@github.com:datalad/testrepo_gh.git',
+    ]):
+        ds = install(source=url, path=opj(path, "clone%i" % i), recursive=True)
+        eq_(len(ds.subdatasets(recursive=True, fulfilled=True)), 2)