Repository: datalad/datalad

Issue:
Unexpected `install/clone` behavior (relpath misinterpretation)
I ran

```
In [1]: import datalad.api as dl                                                             
In [7]: dl.install('~/forrest/collection/phase2', 'dest2')                                   
[INFO   ] Cloning dest2 into '/tmp/~/forrest/collection/phase2' 
Out[7]: <Dataset path=/tmp/~/forrest/collection/phase2>
```

I my case both locations have a dataset, so I expected failure and was surprised by the success. It is mistaking the `~` path for a relative path specification. `clone` has the same issue.

Discussion:

Patch:
diff --git a/appveyor.yml b/appveyor.yml
index 0e53df4bd..ff51b254f 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -92,7 +92,7 @@ test_script:
   # remaining fails: test_archives.test_basic_scenario test_datalad.test_basic_scenario_local_url
   #- python -m nose -s -v datalad.customremotes
   # remaining fails: test_add test_create_test_dataset test_uninstall test_utils
-  - "python -m nose -s -v --with-cov --cover-package datalad datalad.distribution.tests.test_get datalad.distribution.tests.test_create_github datalad.distribution.tests.test_dataset_binding datalad.distribution.tests.test_siblings datalad.distribution.tests.test_update datalad.distribution.tests.test_dataset datalad.distribution.tests.test_publish datalad.local.tests.test_subdataset datalad.distribution.tests.test_clone datalad.distribution.tests.test_create_sibling"
+  - "python -m nose -s -v --with-cov --cover-package datalad datalad.distribution.tests.test_get datalad.distribution.tests.test_create_github datalad.distribution.tests.test_dataset_binding datalad.distribution.tests.test_siblings datalad.distribution.tests.test_update datalad.distribution.tests.test_dataset datalad.distribution.tests.test_publish datalad.local.tests.test_subdataset datalad.distribution.tests.test_create_sibling"
   # remaining fails: test_http
   - "python -m nose -s -v --with-cov --cover-package datalad datalad.downloaders.tests.test_credentials datalad.downloaders.tests.test_providers datalad.downloaders.tests.test_s3"
   # remaining fails: test_add_archive_content test_annotate_paths test_diff test_ls_webui test_run test_save test_utils
diff --git a/datalad/core/distributed/__init__.py b/datalad/core/distributed/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/datalad/core/distributed/clone.py b/datalad/core/distributed/clone.py
new file mode 100644
index 000000000..40d47400d
--- /dev/null
+++ b/datalad/core/distributed/clone.py
@@ -0,0 +1,638 @@
+# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
+# ex: set sts=4 ts=4 sw=4 noet:
+# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
+#
+#   See COPYING file distributed along with the datalad package for the
+#   copyright and license terms.
+#
+# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
+"""Plumbing command for dataset installation"""
+
+
+import logging
+import os
+import re
+from os.path import expanduser
+from collections import OrderedDict
+from urllib.parse import unquote as urlunquote
+
+from datalad.interface.base import Interface
+from datalad.interface.utils import eval_results
+from datalad.interface.base import build_doc
+from datalad.interface.results import get_status_dict
+from datalad.interface.common_opts import (
+    location_description,
+    reckless_opt,
+)
+from datalad.log import log_progress
+from datalad.support.gitrepo import (
+    GitRepo,
+    GitCommandError,
+)
+from datalad.support.annexrepo import AnnexRepo
+from datalad.support.constraints import (
+    EnsureNone,
+    EnsureStr,
+    EnsureKeyChoice,
+)
+from datalad.support.param import Parameter
+from datalad.support.network import (
+    get_local_file_url,
+    URL,
+    RI,
+    DataLadRI,
+    PathRI,
+)
+from datalad.dochelpers import (
+    exc_str,
+    single_or_plural,
+)
+from datalad.utils import (
+    rmtree,
+    assure_bool,
+    knows_annex,
+    Path,
+)
+
+from datalad.distribution.dataset import (
+    Dataset,
+    datasetmethod,
+    resolve_path,
+    require_dataset,
+    EnsureDataset,
+)
+from datalad.distribution.utils import (
+    _get_git_url_from_source,
+    _get_flexible_source_candidates,
+)
+
+__docformat__ = 'restructuredtext'
+
+lgr = logging.getLogger('datalad.core.distributed.clone')
+
+
+@build_doc
+class Clone(Interface):
+    """Obtain a dataset (copy) from a URL or local directory
+
+    The purpose of this command is to obtain a new clone (copy) of a dataset
+    and place it into a not-yet-existing or empty directory. As such `clone`
+    provides a strict subset of the functionality offered by `install`. Only a
+    single dataset can be obtained, and immediate recursive installation of
+    subdatasets is not supported. However, once a (super)dataset is installed
+    via `clone`, any content, including subdatasets can be obtained by a
+    subsequent `get` command.
+
+    Primary differences over a direct `git clone` call are 1) the automatic
+    initialization of a dataset annex (pure Git repositories are equally
+    supported); 2) automatic registration of the newly obtained dataset
+    as a subdataset (submodule), if a parent dataset is specified; and
+    3) support for datalad's resource identifiers and automatic configurable
+    generation of alternative access URL for common cases (such as appending
+    '.git' to the URL in case the accessing the base URL failed).
+
+    || PYTHON >>By default, the command returns a single Dataset instance for
+    an installed dataset, regardless of whether it was newly installed ('ok'
+    result), or found already installed from the specified source ('notneeded'
+    result).<< PYTHON ||
+    """
+    # by default ignore everything but install results
+    # i.e. no "add to super dataset"
+    result_filter = EnsureKeyChoice('action', ('install',))
+    # very frequently this command will yield exactly one installed dataset
+    # spare people the pain of going through a list by default
+    return_type = 'item-or-list'
+    # as discussed in #1409 and #1470, we want to return dataset instances
+    # matching what is actually available after command completion (and
+    # None for any failed dataset installation)
+    result_xfm = 'successdatasets-or-none'
+
+    _examples_ = [
+        dict(text="Install a dataset from Github into the current directory",
+             code_py="clone("
+             "source='https://github.com/datalad-datasets/longnow"
+             "-podcasts.git')",
+             code_cmd="datalad clone "
+             "https://github.com/datalad-datasets/longnow-podcasts.git"),
+        dict(text="Install a dataset into a specific directory",
+             code_py="clone("
+             "source='https://github.com/datalad-datasets/longnow"
+             "-podcasts.git', path='myfavpodcasts')",
+             code_cmd="datalad clone "
+             "https://github.com/datalad-datasets/longnow-podcasts.git "
+             "myfavpodcasts"),
+        dict(text="Install a dataset as a subdataset into the current dataset",
+             code_py="clone(dataset='.', "
+             "source='https://github.com/datalad-datasets/longnow-podcasts.git')",
+             code_cmd="datalad clone -d . "
+             "--source='https://github.com/datalad-datasets/longnow-podcasts.git'"),
+    ]
+
+    _params_ = dict(
+        dataset=Parameter(
+            args=("-d", "--dataset"),
+            doc="""(parent) dataset to clone into. If given, the newly cloned
+            dataset is registered as a subdataset of the parent. Also, if given,
+            relative paths are interpreted as being relative to the parent
+            dataset, and not relative to the working directory.""",
+            constraints=EnsureDataset() | EnsureNone()),
+        source=Parameter(
+            args=("source",),
+            metavar='SOURCE',
+            doc="""URL, DataLad resource identifier, local path or instance of
+            dataset to be cloned""",
+            constraints=EnsureStr() | EnsureNone()),
+        path=Parameter(
+            args=("path",),
+            metavar='PATH',
+            nargs="?",
+            doc="""path to clone into.  If no `path` is provided a
+            destination path will be derived from a source URL
+            similar to :command:`git clone`"""),
+        description=location_description,
+        reckless=reckless_opt,
+    )
+
+    @staticmethod
+    @datasetmethod(name='clone')
+    @eval_results
+    def __call__(
+            source,
+            path=None,
+            dataset=None,
+            description=None,
+            reckless=None):
+        # legacy compatibility
+        if reckless is True:
+            # so that we can forget about how things used to be
+            reckless = 'auto'
+
+        # did we explicitly get a dataset to install into?
+        # if we got a dataset, path will be resolved against it.
+        # Otherwise path will be resolved first.
+        ds = require_dataset(
+            dataset, check_installed=True, purpose='cloning') \
+            if dataset is not None else dataset
+        refds_path = ds.path if ds else None
+
+        if isinstance(source, Dataset):
+            source = source.path
+
+        if source == path:
+            # even if they turn out to be identical after resolving symlinks
+            # and more sophisticated witchcraft, it would still happily say
+            # "it appears to be already installed", so we just catch an
+            # obviously pointless input combination
+            raise ValueError(
+                "clone `source` and destination `path` are identical [{}]. "
+                "If you are trying to add a subdataset simply use `save`".format(
+                    path))
+
+        if path is not None:
+            path = resolve_path(path, dataset)
+
+        # Possibly do conversion from source into a git-friendly url
+        # luckily GitRepo will undo any fancy file:/// url to make use of Git's
+        # optimization for local clones....
+        source_url = source
+        source_ = _get_git_url_from_source(source)
+        lgr.debug("Resolved clone source from '%s' to '%s'",
+                  source, source_)
+        source = source_
+
+        # derive target from source:
+        if path is None:
+            # we got nothing but a source. do something similar to git clone
+            # and derive the path from the source and continue
+            path = _get_installationpath_from_url(source)
+            # since this is a relative `path`, resolve it:
+            path = resolve_path(path, dataset)
+            lgr.debug("Determined clone target path from source")
+        lgr.debug("Resolved clone target path to: '%s'", path)
+
+        # there is no other way -- my intoxicated brain tells me
+        assert(path is not None)
+
+        result_props = dict(
+            action='install',
+            logger=lgr,
+            refds=refds_path,
+            source_url=source_url)
+
+        try:
+            # this will implicitly cause pathlib to run a bunch of checks
+            # whether the present path makes any sense on the platform
+            # we are running on -- we don't care if the path actually
+            # exists at this point, but we want to abort early if the path
+            # spec is determined to be useless
+            path.exists()
+        except OSError as e:
+            yield get_status_dict(
+                status='error',
+                path=path,
+                message=('cannot handle target path: %s', exc_str(e)),
+                **result_props)
+            return
+
+        destination_dataset = Dataset(path)
+        result_props['ds'] = destination_dataset
+
+        if ds is not None and ds.pathobj not in path.parents:
+            yield get_status_dict(
+                status='error',
+                message=("clone target path '%s' not in specified target dataset '%s'",
+                         path, ds),
+                **result_props)
+            return
+
+        # perform the actual cloning operation
+        yield from clone_dataset(
+            [source],
+            destination_dataset,
+            reckless,
+            description,
+            result_props,
+        )
+
+        if ds is not None:
+            # we created a dataset in another dataset
+            # -> make submodule
+            for r in ds.save(
+                    path,
+                    return_type='generator',
+                    result_filter=None,
+                    result_xfm=None,
+                    on_failure='ignore'):
+                yield r
+
+
+def clone_dataset(
+        srcs,
+        destds,
+        reckless=None,
+        description=None,
+        result_props=None):
+    """Internal helper to perform cloning without sanity checks (assumed done)
+
+    This helper does not handle any saving of subdataset modification or adding
+    in a superdataset.
+
+    Parameters
+    ----------
+    srcs : list
+      Any suitable clone source specifications (paths, URLs)
+    destds : Dataset
+      Dataset instance for the clone destination
+    reckless : {None, 'auto'}, optional
+      Mode switch to put cloned dataset into throw-away configurations, i.e.
+      sacrifice data safety for performance or resource footprint.
+    description : str, optional
+      Location description for the annex of the dataset clone (if there is any).
+    result_props : dict, optional
+      Default properties for any yielded result, passed on to get_status_dict().
+
+    Yields
+    ------
+    dict
+      DataLad result records
+    """
+    if not result_props:
+        # in case the caller had no specific idea on how results should look
+        # like, provide sensible defaults
+        result_props = dict(
+            action='install',
+            logger=lgr,
+            ds=destds,
+        )
+
+    dest_path = destds.pathobj
+
+    # generate candidate URLs from source argument to overcome a few corner cases
+    # and hopefully be more robust than git clone
+    candidate_sources = [
+        s for src in srcs
+        for s in _get_flexible_source_candidates(src)
+    ]
+
+    # important test! based on this `rmtree` will happen below after failed clone
+    dest_path_existed = dest_path.exists()
+    if dest_path_existed and any(dest_path.iterdir()):
+        if destds.is_installed():
+            # check if dest was cloned from the given source before
+            # this is where we would have installed this from
+            # this is where it was actually installed from
+            track_name, track_url = _get_tracking_source(destds)
+            try:
+                # this will get us track_url in system native path conventions,
+                # whenever it is a path (and not a URL)
+                # this is needed to match it to any potentially incoming local
+                # source path in the 'notneeded' test below
+                track_path = str(Path(track_url))
+            except Exception:
+                # this should never happen, because Path() will let any non-path stringification
+                # pass through unmodified, but we do not want any potential crash due to
+                # pathlib behavior changes
+                lgr.debug("Unexpected behavior of pathlib!")
+                track_path = None
+            for src in candidate_sources:
+                if track_url == src \
+                        or get_local_file_url(track_url, compatibility='git') == src \
+                        or track_path == expanduser(src):
+                    yield get_status_dict(
+                        status='notneeded',
+                        message=("dataset %s was already cloned from '%s'",
+                                 destds,
+                                 src),
+                        **result_props)
+                    return
+        # anything else is an error
+        yield get_status_dict(
+            status='error',
+            message='target path already exists and not empty, refuse to clone into target path',
+            **result_props)
+        return
+
+    log_progress(
+        lgr.info,
+        'cloneds',
+        'Cloning dataset to %s', destds,
+        total=len(candidate_sources),
+        label='Clone attempt',
+        unit=' Candidate locations',
+    )
+    error_msgs = OrderedDict()  # accumulate all error messages formatted per each url
+    for isource_, source_ in enumerate(candidate_sources):
+        try:
+            log_progress(
+                lgr.info,
+                'cloneds',
+                'Attempting to clone from %s to %s', source_, dest_path,
+                update=1,
+                increment=True)
+            # TODO for now GitRepo.clone() cannot handle Path instances, and PY35
+            # doesn't make it happen seemlessly
+            GitRepo.clone(path=str(dest_path), url=source_, create=True)
+            break  # do not bother with other sources if succeeded
+        except GitCommandError as e:
+            error_msgs[source_] = e
+            lgr.debug("Failed to clone from URL: %s (%s)",
+                      source_, exc_str(e))
+            if dest_path.exists():
+                lgr.debug("Wiping out unsuccessful clone attempt at: %s",
+                          dest_path)
+                # We must not just rmtree since it might be curdir etc
+                # we should remove all files/directories under it
+                # TODO stringification can be removed once patlib compatible
+                # or if PY35 is no longer supported
+                rmtree(str(dest_path), children_only=dest_path_existed)
+            # Whenever progress reporting is enabled, as it is now,
+            # we end up without e.stderr since it is "processed" out by
+            # GitPython/our progress handler.
+            e_stderr = e.stderr
+            from datalad.support.gitrepo import GitPythonProgressBar
+            if not e_stderr and GitPythonProgressBar._last_error_lines:
+                e_stderr = os.linesep.join(GitPythonProgressBar._last_error_lines)
+            if 'could not create work tree' in e_stderr.lower():
+                # this cannot be fixed by trying another URL
+                re_match = re.match(r".*fatal: (.*)$", e_stderr,
+                                    flags=re.MULTILINE | re.DOTALL)
+                # cancel progress bar
+                log_progress(
+                    lgr.info,
+                    'cloneds',
+                    'Completed clone attempts for %s', destds
+                )
+                yield get_status_dict(
+                    status='error',
+                    message=re_match.group(1) if re_match else "stderr: " + e_stderr,
+                    **result_props)
+                return
+    log_progress(
+        lgr.info,
+        'cloneds',
+        'Completed clone attempts for %s', destds
+    )
+
+    if not destds.is_installed():
+        if len(error_msgs):
+            if all(not e.stdout and not e.stderr for e in error_msgs.values()):
+                # there is nothing we can learn from the actual exception,
+                # the exit code is uninformative, the command is predictable
+                error_msg = "Failed to clone from all attempted sources: %s"
+                error_args = list(error_msgs.keys())
+            else:
+                error_msg = "Failed to clone from any candidate source URL. " \
+                            "Encountered errors per each url were:\n- %s"
+                error_args = '\n- '.join(
+                    '{}\n  {}'.format(url, exc_str(exc))
+                    for url, exc in error_msgs.items()
+                )
+        else:
+            # yoh: Not sure if we ever get here but I felt that there could
+            #      be a case when this might happen and original error would
+            #      not be sufficient to troubleshoot what is going on.
+            error_msg = "Awkward error -- we failed to clone properly. " \
+                        "Although no errors were encountered, target " \
+                        "dataset at %s seems to be not fully installed. " \
+                        "The 'succesful' source was: %s"
+            error_args = (destds.path, source_)
+        yield get_status_dict(
+            status='error',
+            message=(error_msg, error_args),
+            **result_props)
+        return
+
+    yield from _handle_possible_annex_dataset(
+        destds,
+        reckless,
+        description)
+
+    # yield successful clone of the base dataset now, as any possible
+    # subdataset clone down below will not alter the Git-state of the
+    # parent
+    yield get_status_dict(status='ok', **result_props)
+
+
+def _handle_possible_annex_dataset(ds, reckless, description=None):
+    """If ds "knows annex" -- annex init it, set into reckless etc
+
+    Provides additional tune up to a possibly an annex repo, e.g.
+    "enables" reckless mode, sets up description
+    """
+    # in any case check whether we need to annex-init the installed thing:
+    if not knows_annex(ds.path):
+        # not for us
+        return
+
+    # init annex when traces of a remote annex can be detected
+    if reckless:
+        lgr.debug(
+            "Instruct annex to hardlink content in %s from local "
+            "sources, if possible (reckless)", ds.path)
+        ds.config.add(
+            'annex.hardlink', 'true', where='local', reload=True)
+
+    # we have just cloned the repo, so it has 'origin', configure any
+    # reachable origin of origins
+    yield from configure_origins(ds, ds)
+
+    lgr.debug("Initializing annex repo at %s", ds.path)
+    # Note, that we cannot enforce annex-init via AnnexRepo().
+    # If such an instance already exists, its __init__ will not be executed.
+    # Therefore do quick test once we have an object and decide whether to call its _init().
+    #
+    # Additionally, call init if we need to add a description (see #1403),
+    # since AnnexRepo.__init__ can only do it with create=True
+    repo = AnnexRepo(ds.path, init=True)
+    if not repo.is_initialized() or description:
+        repo._init(description=description)
+    if reckless:
+        repo._run_annex_command('untrust', annex_options=['here'])
+
+    srs = {True: [], False: []}  # special remotes by "autoenable" key
+    remote_uuids = None  # might be necessary to discover known UUIDs
+
+    # Note: The purpose of this function is to inform the user. So if something
+    # looks misconfigured, we'll warn and move on to the next item.
+    for uuid, config in repo.get_special_remotes().items():
+        sr_name = config.get('name', None)
+        if sr_name is None:
+            lgr.warning(
+                'Ignoring special remote %s because it does not have a name. '
+                'Known information: %s',
+                uuid, config)
+            continue
+        sr_autoenable = config.get('autoenable', False)
+        try:
+            sr_autoenable = assure_bool(sr_autoenable)
+        except ValueError:
+            lgr.warning(
+                'Failed to process "autoenable" value %r for sibling %s in '
+                'dataset %s as bool.  You might need to enable it later '
+                'manually and/or fix it up to avoid this message in the future.',
+                sr_autoenable, sr_name, ds.path)
+            continue
+
+        # determine whether there is a registered remote with matching UUID
+        if uuid:
+            if remote_uuids is None:
+                remote_uuids = {
+                    # Check annex-config-uuid first. For sameas annex remotes,
+                    # this will point to the UUID for the configuration (i.e.
+                    # the key returned by get_special_remotes) rather than the
+                    # shared UUID.
+                    (repo.config.get('remote.%s.annex-config-uuid' % r) or
+                     repo.config.get('remote.%s.annex-uuid' % r))
+                    for r in repo.get_remotes()
+                }
+            if uuid not in remote_uuids:
+                srs[sr_autoenable].append(sr_name)
+
+    if srs[True]:
+        lgr.debug(
+            "configuration for %s %s added because of autoenable,"
+            " but no UUIDs for them yet known for dataset %s",
+            # since we are only at debug level, we could call things their
+            # proper names
+            single_or_plural("special remote", "special remotes", len(srs[True]), True),
+            ", ".join(srs[True]),
+            ds.path
+        )
+
+    if srs[False]:
+        # if has no auto-enable special remotes
+        lgr.info(
+            'access to %s %s not auto-enabled, enable with:\n\t\tdatalad siblings -d "%s" enable -s %s',
+            # but since humans might read it, we better confuse them with our
+            # own terms!
+            single_or_plural("dataset sibling", "dataset siblings", len(srs[False]), True),
+            ", ".join(srs[False]),
+            ds.path,
+            srs[False][0] if len(srs[False]) == 1 else "SIBLING",
+        )
+
+
+def configure_origins(cfgds, probeds, label=None):
+    """Configure any discoverable local dataset 'origin' sibling as a remote
+
+    Parameters
+    ----------
+    cfgds : Dataset
+      Dataset to receive the remote configurations
+    probeds : Dataset
+      Dataset to start looking for 'origin' remotes. May be identical with `cfgds`.
+    label : int, optional
+      Each discovered 'origin' will be configured as a remote under the name
+      'origin-<label>'. If no label is given, '2' will be used by default, given that
+      there is typically a 'origin' remote already.
+    """
+    if label is None:
+        label = 2
+    # let's look at the URL for that remote and see if it is a local
+    # dataset
+    origin_url = probeds.config.get('remote.origin.url')
+    if origin_url and cfgds.config.obtain(
+            'datalad.install.inherit-local-origin',
+            default=True) and isinstance(RI(origin_url), PathRI):
+        # given the clone source is a local dataset, we can have a
+        # cheap look at it, and configure its own 'origin' as a remote
+        # (if there is any), and benefit from additional annex availability
+        originorigin_ds = Dataset(origin_url)
+        originorigin_url = originorigin_ds.config.get('remote.origin.url')
+        if originorigin_url:
+            yield from cfgds.siblings(
+                'configure',
+                # no chance for config, can only be the second configured remote
+                name='origin-{}'.format(label),
+                url=originorigin_url,
+                # fetch to get all annex info
+                fetch=True,
+                result_renderer='disabled',
+                on_failure='ignore',
+            )
+        # and dive deeper
+        yield from configure_origins(cfgds, originorigin_ds, label=label + 1)
+
+
+def _get_tracking_source(ds):
+    """Returns name and url of a potential configured source
+    tracking remote"""
+    vcs = ds.repo
+    # if we have a remote, let's check the location of that remote
+    # for the presence of the desired submodule
+
+    remote_name, tracking_branch = vcs.get_tracking_branch()
+    if not remote_name and isinstance(vcs, AnnexRepo):
+        # maybe cloned from a source repo that was in adjusted mode
+        # https://github.com/datalad/datalad/issues/3969
+        remote_name, tracking_branch = vcs.get_tracking_branch(
+            corresponding=False)
+    # TODO: better default `None`? Check where we might rely on '':
+    remote_url = ''
+    if remote_name:
+        remote_url = vcs.get_remote_url(remote_name, push=False)
+
+    return remote_name, remote_url
+
+
+def _get_installationpath_from_url(url):
+    """Returns a relative path derived from the trailing end of a URL
+
+    This can be used to determine an installation path of a Dataset
+    from a URL, analog to what `git clone` does.
+    """
+    ri = RI(url)
+    if isinstance(ri, (URL, DataLadRI)):  # decode only if URL
+        path = ri.path.rstrip('/')
+        path = urlunquote(path) if path else ri.hostname
+        if '/' in path:
+            path = path.split('/')
+            if path[-1] == '.git':
+                path = path[-2]
+            else:
+                path = path[-1]
+    else:
+        path = Path(url).parts[-1]
+    if path.endswith('.git'):
+        path = path[:-4]
+    return path
diff --git a/datalad/distribution/clone.py b/datalad/distribution/clone.py
index 6310b0d86..c4b9e10c8 100644
--- a/datalad/distribution/clone.py
+++ b/datalad/distribution/clone.py
@@ -6,472 +6,21 @@
 #   copyright and license terms.
 #
 # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
-"""Plumbing command for dataset installation"""
+"""Compatibility module for datalad.core.distributed.clone.
 
-
-import logging
-import os
-import re
-from collections import OrderedDict
-from urllib.parse import unquote as urlunquote
-
-from datalad.interface.base import Interface
-from datalad.interface.utils import eval_results
-from datalad.interface.base import build_doc
-from datalad.interface.results import get_status_dict
-from datalad.interface.common_opts import (
-    location_description,
-    reckless_opt,
-)
-from datalad.support.gitrepo import (
-    GitRepo,
-    GitCommandError,
-)
-from datalad.support.annexrepo import AnnexRepo
-from datalad.support.constraints import (
-    EnsureNone,
-    EnsureStr,
-    EnsureKeyChoice,
-)
-from datalad.support.param import Parameter
-from datalad.support.network import (
-    get_local_file_url,
-    URL,
-    RI,
-    DataLadRI,
-)
-from datalad.dochelpers import (
-    exc_str,
-    single_or_plural,
-)
-from datalad.utils import (
-    rmtree,
-    assure_list,
-    assure_bool,
-    knows_annex,
-)
-
-from datalad.distribution.dataset import (
-    Dataset,
-    datasetmethod,
-    resolve_path,
-    require_dataset,
-    EnsureDataset,
-)
-from datalad.distribution.utils import (
-    _get_git_url_from_source,
-    _get_flexible_source_candidates,
-)
+Use datalad.core.distributed.clone instead."""
 
 __docformat__ = 'restructuredtext'
 
-lgr = logging.getLogger('datalad.distribution.clone')
-
-
-@build_doc
-class Clone(Interface):
-    """Obtain a dataset copy from a URL or local source (path)
-
-    The purpose of this command is to obtain a new clone (copy) of a dataset
-    and place it into a not-yet-existing or empty directory. As such `clone`
-    provides a strict subset of the functionality offered by `install`. Only a
-    single dataset can be obtained, recursion is not supported. However, once
-    installed, arbitrary dataset components can be obtained via a subsequent
-    `get` command.
-
-    Primary differences over a direct `git clone` call are 1) the automatic
-    initialization of a dataset annex (pure Git repositories are equally
-    supported); 2) automatic registration of the newly obtained dataset
-    as a subdataset (submodule), if a parent dataset is specified;
-    3) support for datalad's resource identifiers and automatic generation of
-    alternative access URL for common cases (such as appending '.git' to the
-    URL in case the accessing the base URL failed); and 4) ability to
-    take additional alternative source locations as an argument.
-    """
-    # by default ignore everything but install results
-    # i.e. no "add to super dataset"
-    result_filter = EnsureKeyChoice('action', ('install',))
-
-    _params_ = dict(
-        dataset=Parameter(
-            args=("-d", "--dataset"),
-            doc="""(parent) dataset to clone into. If given, the newly cloned
-            dataset is registered as a subdataset of the parent. Also, if given,
-            relative paths are interpreted as being relative to the parent
-            dataset, and not relative to the working directory.""",
-            constraints=EnsureDataset() | EnsureNone()),
-        source=Parameter(
-            args=("source",),
-            metavar='SOURCE',
-            doc="""URL, DataLad resource identifier, local path or instance of
-            dataset to be cloned""",
-            constraints=EnsureStr() | EnsureNone()),
-        path=Parameter(
-            args=("path",),
-            metavar='PATH',
-            nargs="?",
-            doc="""path to clone into.  If no `path` is provided a
-            destination path will be derived from a source URL
-            similar to :command:`git clone`"""),
-        description=location_description,
-        reckless=reckless_opt,
-        alt_sources=Parameter(
-            args=('--alternative-sources',),
-            dest='alt_sources',
-            metavar='SOURCE',
-            nargs='+',
-            doc="""Alternative sources to be tried if a dataset cannot
-            be obtained from the main `source`""",
-            constraints=EnsureStr() | EnsureNone()),
-    )
-
-    @staticmethod
-    @datasetmethod(name='clone')
-    @eval_results
-    def __call__(
-            source,
-            path=None,
-            dataset=None,
-            description=None,
-            reckless=False,
-            alt_sources=None):
-
-        # did we explicitly get a dataset to install into?
-        # if we got a dataset, path will be resolved against it.
-        # Otherwise path will be resolved first.
-        ds = require_dataset(
-            dataset, check_installed=True, purpose='cloning') \
-            if dataset is not None else dataset
-        refds_path = ds.path if ds else None
-
-        if isinstance(source, Dataset):
-            source = source.path
-
-        if source == path:
-            # even if they turn out to be identical after resolving symlinks
-            # and more sophisticated witchcraft, it would still happily say
-            # "it appears to be already installed", so we just catch an
-            # obviously pointless input combination
-            raise ValueError(
-                "clone `source` and destination `path` are identical [{}]. "
-                "If you are trying to add a subdataset simply use `add`".format(
-                    path))
-
-        if path is not None:
-            path = resolve_path(path, dataset)
-
-        # Possibly do conversion from source into a git-friendly url
-        # luckily GitRepo will undo any fancy file:/// url to make use of Git's
-        # optimization for local clones....
-        source_url = source
-        source_ = _get_git_url_from_source(source)
-        lgr.debug("Resolved clone source from '%s' to '%s'",
-                  source, source_)
-        source = source_
-
-        # derive target from source:
-        if path is None:
-            # we got nothing but a source. do something similar to git clone
-            # and derive the path from the source and continue
-            path = _get_installationpath_from_url(source)
-            # since this is a relative `path`, resolve it:
-            path = resolve_path(path, dataset)
-            lgr.debug("Determined clone target path from source")
-        lgr.debug("Resolved clone target path to: '%s'", path)
-
-        # there is no other way -- my intoxicated brain tells me
-        assert(path is not None)
-
-        status_kwargs = dict(
-            action='install',
-            logger=lgr,
-            refds=refds_path,
-            source_url=source_url)
+import warnings
 
-        try:
-            # this will implicitly cause pathlib to run a bunch of checks
-            # whether the present path makes any sense on the platform
-            # we are running on -- we don't care if the path actually
-            # exists at this point, but we want to abort early if the path
-            # spec is determined to be useless
-            path.exists()
-        except OSError as e:
-            yield get_status_dict(
-                status='error',
-                path=path,
-                message=('cannot handle target path: %s', exc_str(e)),
-                **status_kwargs)
-            return
-
-        destination_dataset = Dataset(path)
-        status_kwargs['ds'] = destination_dataset
-        dest_path = path
-
-        # important test! based on this `rmtree` will happen below after failed clone
-        if dest_path.exists() and any(dest_path.iterdir()):
-            if destination_dataset.is_installed():
-                # check if dest was cloned from the given source before
-                # this is where we would have installed this from
-                guessed_sources = _get_flexible_source_candidates(
-                    source, dest_path)
-                # this is where it was actually installed from
-                track_name, track_url = _get_tracking_source(destination_dataset)
-                if track_url in guessed_sources or \
-                        get_local_file_url(
-                            track_url, compatibility='git') in guessed_sources:
-                    yield get_status_dict(
-                        status='notneeded',
-                        message=("dataset %s was already cloned from '%s'",
-                                 destination_dataset,
-                                 source),
-                        **status_kwargs)
-                    return
-            # anything else is an error
-            yield get_status_dict(
-                status='error',
-                message='target path already exists and not empty, refuse to clone into target path',
-                **status_kwargs)
-            return
-
-        if ds is not None and ds.pathobj not in dest_path.parents:
-            yield get_status_dict(
-                status='error',
-                message=("clone target path '%s' not in specified target dataset '%s'",
-                         dest_path, ds),
-                **status_kwargs)
-            return
-
-        # generate candidate URLs from source argument to overcome a few corner cases
-        # and hopefully be more robust than git clone
-        candidate_sources = []
-        # combine all given sources (incl. alternatives), maintain order
-        for s in [source] + assure_list(alt_sources):
-            candidate_sources.extend(_get_flexible_source_candidates(s))
-        candidates_str = \
-            " [%d other candidates]" % (len(candidate_sources) - 1) \
-            if len(candidate_sources) > 1 \
-            else ''
-        lgr.info("Cloning %s%s into '%s'",
-                 source, candidates_str, dest_path)
-        dest_path_existed = dest_path.exists()
-        error_msgs = OrderedDict()  # accumulate all error messages formatted per each url
-        for isource_, source_ in enumerate(candidate_sources):
-            try:
-                lgr.debug("Attempting to clone %s (%d out of %d candidates) to '%s'",
-                          source_, isource_ + 1, len(candidate_sources), dest_path)
-                # TODO for now GitRepo.clone() cannot handle Path instances, and PY35
-                # doesn't make it happen seemlessly
-                GitRepo.clone(path=str(dest_path), url=source_, create=True)
-                break  # do not bother with other sources if succeeded
-            except GitCommandError as e:
-                error_msgs[source_] = e
-                lgr.debug("Failed to clone from URL: %s (%s)",
-                          source_, exc_str(e))
-                if dest_path.exists():
-                    lgr.debug("Wiping out unsuccessful clone attempt at: %s",
-                              dest_path)
-                    # We must not just rmtree since it might be curdir etc
-                    # we should remove all files/directories under it
-                    # TODO stringification can be removed once patlib compatible
-                    # or if PY35 is no longer supported
-                    rmtree(str(dest_path), children_only=dest_path_existed)
-                # Whenever progress reporting is enabled, as it is now,
-                # we end up without e.stderr since it is "processed" out by
-                # GitPython/our progress handler.
-                e_stderr = e.stderr
-                from datalad.support.gitrepo import GitPythonProgressBar
-                if not e_stderr and GitPythonProgressBar._last_error_lines:
-                    e_stderr = os.linesep.join(GitPythonProgressBar._last_error_lines)
-                if 'could not create work tree' in e_stderr.lower():
-                    # this cannot be fixed by trying another URL
-                    re_match = re.match(r".*fatal: (.*)$", e_stderr,
-                                        flags=re.MULTILINE | re.DOTALL)
-                    yield get_status_dict(
-                        status='error',
-                        message=re_match.group(1) if re_match else "stderr: " + e_stderr,
-                        **status_kwargs)
-                    return
-
-        if not destination_dataset.is_installed():
-            if len(error_msgs):
-                if all(not e.stdout and not e.stderr for e in error_msgs.values()):
-                    # there is nothing we can learn from the actual exception,
-                    # the exit code is uninformative, the command is predictable
-                    error_msg = "Failed to clone from all attempted sources: %s"
-                    error_args = list(error_msgs.keys())
-                else:
-                    error_msg = "Failed to clone from any candidate source URL. " \
-                                "Encountered errors per each url were:\n- %s"
-                    error_args = '\n- '.join(
-                        '{}\n  {}'.format(url, exc_str(exc))
-                        for url, exc in error_msgs.items()
-                    )
-            else:
-                # yoh: Not sure if we ever get here but I felt that there could
-                #      be a case when this might happen and original error would
-                #      not be sufficient to troubleshoot what is going on.
-                error_msg = "Awkward error -- we failed to clone properly. " \
-                            "Although no errors were encountered, target " \
-                            "dataset at %s seems to be not fully installed. " \
-                            "The 'succesful' source was: %s"
-                error_args = (destination_dataset.path, source_)
-            yield get_status_dict(
-                status='error',
-                message=(error_msg, error_args),
-                **status_kwargs)
-            return
-
-        if ds is not None:
-            # we created a dataset in another dataset
-            # -> make submodule
-            for r in ds.save(
-                    dest_path,
-                    return_type='generator',
-                    result_filter=None,
-                    result_xfm=None,
-                    on_failure='ignore'):
-                yield r
-
-        _handle_possible_annex_dataset(
-            destination_dataset,
-            reckless,
-            description=description)
-
-        # yield successful clone of the base dataset now, as any possible
-        # subdataset clone down below will not alter the Git-state of the
-        # parent
-        yield get_status_dict(status='ok', **status_kwargs)
-
-
-def _handle_possible_annex_dataset(dataset, reckless, description=None):
-    """If dataset "knows annex" -- annex init it, set into reckless etc
-
-    Provides additional tune up to a possibly an annex repo, e.g.
-    "enables" reckless mode, sets up description
-    """
-    # in any case check whether we need to annex-init the installed thing:
-    if not knows_annex(dataset.path):
-        # not for us
-        return
-
-    # init annex when traces of a remote annex can be detected
-    if reckless:
-        lgr.debug(
-            "Instruct annex to hardlink content in %s from local "
-            "sources, if possible (reckless)", dataset.path)
-        dataset.config.add(
-            'annex.hardlink', 'true', where='local', reload=True)
-    lgr.debug("Initializing annex repo at %s", dataset.path)
-    # Note, that we cannot enforce annex-init via AnnexRepo().
-    # If such an instance already exists, its __init__ will not be executed.
-    # Therefore do quick test once we have an object and decide whether to call its _init().
-    #
-    # Additionally, call init if we need to add a description (see #1403),
-    # since AnnexRepo.__init__ can only do it with create=True
-    repo = AnnexRepo(dataset.path, init=True)
-    if not repo.is_initialized() or description:
-        repo._init(description=description)
-    if reckless:
-        repo._run_annex_command('untrust', annex_options=['here'])
-
-    srs = {True: [], False: []}  # special remotes by "autoenable" key
-    remote_uuids = None  # might be necessary to discover known UUIDs
-
-    # Note: The purpose of this function is to inform the user. So if something
-    # looks misconfigured, we'll warn and move on to the next item.
-    for uuid, config in repo.get_special_remotes().items():
-        sr_name = config.get('name', None)
-        if sr_name is None:
-            lgr.warning(
-                'Ignoring special remote %s because it does not have a name. '
-                'Known information: %s',
-                uuid, config)
-            continue
-        sr_autoenable = config.get('autoenable', False)
-        try:
-            sr_autoenable = assure_bool(sr_autoenable)
-        except ValueError:
-            lgr.warning(
-                'Failed to process "autoenable" value %r for sibling %s in '
-                'dataset %s as bool.  You might need to enable it later '
-                'manually and/or fix it up to avoid this message in the future.',
-                sr_autoenable, sr_name, dataset.path)
-            continue
-
-        # determine whether there is a registered remote with matching UUID
-        if uuid:
-            if remote_uuids is None:
-                remote_uuids = {
-                    # Check annex-config-uuid first. For sameas annex remotes,
-                    # this will point to the UUID for the configuration (i.e.
-                    # the key returned by get_special_remotes) rather than the
-                    # shared UUID.
-                    (repo.config.get('remote.%s.annex-config-uuid' % r) or
-                     repo.config.get('remote.%s.annex-uuid' % r))
-                    for r in repo.get_remotes()
-                }
-            if uuid not in remote_uuids:
-                srs[sr_autoenable].append(sr_name)
-
-    if srs[True]:
-        lgr.debug(
-            "configuration for %s %s added because of autoenable,"
-            " but no UUIDs for them yet known for dataset %s",
-            # since we are only at debug level, we could call things their
-            # proper names
-            single_or_plural("special remote", "special remotes", len(srs[True]), True),
-            ", ".join(srs[True]),
-            dataset.path
-        )
-
-    if srs[False]:
-        # if has no auto-enable special remotes
-        lgr.info(
-            'access to %s %s not auto-enabled, enable with:\n\t\tdatalad siblings -d "%s" enable -s %s',
-            # but since humans might read it, we better confuse them with our
-            # own terms!
-            single_or_plural("dataset sibling", "dataset siblings", len(srs[False]), True),
-            ", ".join(srs[False]),
-            dataset.path,
-            srs[False][0] if len(srs[False]) == 1 else "SIBLING",
-        )
-
-
-def _get_tracking_source(ds):
-    """Returns name and url of a potential configured source
-    tracking remote"""
-    vcs = ds.repo
-    # if we have a remote, let's check the location of that remote
-    # for the presence of the desired submodule
-
-    remote_name, tracking_branch = vcs.get_tracking_branch()
-    # TODO: better default `None`? Check where we might rely on '':
-    remote_url = ''
-    if remote_name:
-        remote_url = vcs.get_remote_url(remote_name, push=False)
-
-    return remote_name, remote_url
-
-
-def _get_installationpath_from_url(url):
-    """Returns a relative path derived from the trailing end of a URL
+from datalad.core.distributed.clone import (
+    Clone,
+    _get_installationpath_from_url,
+    _handle_possible_annex_dataset,
+    _get_tracking_source,
+)
 
-    This can be used to determine an installation path of a Dataset
-    from a URL, analog to what `git clone` does.
-    """
-    ri = RI(url)
-    if isinstance(ri, (URL, DataLadRI)):  # decode only if URL
-        path = ri.path.rstrip('/')
-        path = urlunquote(path) if path else ri.hostname
-    else:
-        path = url
-    path = path.rstrip('/')
-    if '/' in path:
-        path = path.split('/')
-        if path[-1] == '.git':
-            path = path[-2]
-        else:
-            path = path[-1]
-    if path.endswith('.git'):
-        path = path[:-4]
-    return path
+warnings.warn("datalad.distribution.clone is obsolete. "
+              "Use datalad.core.distributed.clone module instead",
+              DeprecationWarning)
diff --git a/datalad/distribution/get.py b/datalad/distribution/get.py
index e19a628e4..20086d0da 100644
--- a/datalad/distribution/get.py
+++ b/datalad/distribution/get.py
@@ -69,7 +69,7 @@ from datalad.distribution.dataset import (
     datasetmethod,
     require_dataset,
 )
-from datalad.distribution.clone import Clone
+from datalad.core.distributed.clone import clone_dataset
 from datalad.distribution.utils import _get_flexible_source_candidates
 
 __docformat__ = 'restructuredtext'
@@ -253,20 +253,9 @@ def _install_subds_from_flexible_source(ds, sm, **kwargs):
         )
         return
 
-    # now loop over all candidates and try to clone
-    for res in Clone.__call__(
-            clone_urls[0],
-            path=dest_path,
-            # pretend no parent -- we don't want clone to add to ds
-            # because this is a submodule already!
-            dataset=None,
-            # if we have more than one source, pass as alternatives
-            alt_sources=clone_urls[1:],
-            result_xfm=None,
-            # we yield all an have the caller decide
-            on_failure='ignore',
-            result_renderer='disabled',
-            return_type='generator',
+    for res in clone_dataset(
+            clone_urls,
+            Dataset(dest_path),
             **kwargs):
         # make sure to fix a detached HEAD before yielding the install success
         # result. The resetting of the branch would undo any change done
@@ -500,7 +489,8 @@ def _install_targetpath(
         # yield immediately so errors could be acted upon
         # outside, before we continue
         res.update(
-            action='get',
+            # do not override reported action, could be anything
+            #action='get',
             contains=[Path(res['path'])],
         )
         yield res
@@ -641,7 +631,7 @@ class Get(Interface):
             recursion_limit=None,
             get_data=True,
             description=None,
-            reckless=False,
+            reckless=None,
             jobs='auto',
     ):
         refds_path = Interface.get_refds_path(dataset)
diff --git a/datalad/distribution/install.py b/datalad/distribution/install.py
index 9991079d0..d19505f37 100644
--- a/datalad/distribution/install.py
+++ b/datalad/distribution/install.py
@@ -49,7 +49,7 @@ from datalad.distribution.dataset import (
     EnsureDataset,
 )
 from datalad.distribution.get import Get
-from datalad.distribution.clone import Clone
+from datalad.core.distributed.clone import Clone
 
 __docformat__ = 'restructuredtext'
 
@@ -172,7 +172,7 @@ class Install(Interface):
             recursive=False,
             recursion_limit=None,
             save=True,
-            reckless=False,
+            reckless=None,
             jobs="auto"):
 
         # normalize path argument to be equal when called from cmdline and
diff --git a/datalad/interface/__init__.py b/datalad/interface/__init__.py
index ab252cdd1..002743b74 100644
--- a/datalad/interface/__init__.py
+++ b/datalad/interface/__init__.py
@@ -78,7 +78,7 @@ _group_plumbing = (
     'Plumbing commands',
     [
         ('datalad.interface.annotate_paths', 'AnnotatePaths', 'annotate-paths'),
-        ('datalad.distribution.clone', 'Clone'),
+        ('datalad.core.distributed.clone', 'Clone'),
         ('datalad.distribution.create_test_dataset', 'CreateTestDataset',
          'create-test-dataset'),
         ('datalad.core.local.status', 'Status', 'status'),
diff --git a/datalad/interface/base.py b/datalad/interface/base.py
index ff3685d69..a5867b646 100644
--- a/datalad/interface/base.py
+++ b/datalad/interface/base.py
@@ -659,8 +659,11 @@ class Interface(object):
                             p for p in param.constraints._allowed
                             # in the cmdline None pretty much means
                             # don't give the options, so listing it
-                            # doesn't make sense
-                            if p is not None)
+                            # doesn't make sense. Moreover, any non-string
+                            # value cannot be given and very likely only
+                            # serves a special purpose in the Python API
+                            # or implementation details
+                            if isinstance(p, str))
             if defaults_idx >= 0:
                 help += " [Default: %r]" % (defaults[defaults_idx],)
             # create the parameter, using the constraint instance for type
diff --git a/datalad/interface/common_cfg.py b/datalad/interface/common_cfg.py
index 645b072d8..04a2d193b 100644
--- a/datalad/interface/common_cfg.py
+++ b/datalad/interface/common_cfg.py
@@ -337,4 +337,15 @@ definitions = {
         'default': 'auto',
         'type': EnsureChoice('on', 'off', 'auto'),
     },
+    'datalad.install.inherit-local-origin': {
+        'ui': ('question', {
+            'title': 'Inherit local origin of dataset source',
+            'text': "If enabled, a local 'origin' remote of a local dataset "
+                    "clone source is configured as an 'origin-2' remote "
+                    "to make its annex automatically available. The process "
+                    "is repeated recursively for any further qualifying "
+                    "'origin' dataset thereof."}),
+        'default': True,
+        'type': EnsureBool(),
+    },
 }
diff --git a/datalad/interface/common_opts.py b/datalad/interface/common_opts.py
index d2da17f0d..3da81bc24 100644
--- a/datalad/interface/common_opts.py
+++ b/datalad/interface/common_opts.py
@@ -127,7 +127,11 @@ message_file_opt = Parameter(
 
 reckless_opt = Parameter(
     args=("--reckless",),
-    action="store_true",
+    # if no specific mode is given, set to auto
+    const='auto',
+    nargs='?',
+    # boolean types only for backward compatibility
+    constraints=EnsureChoice(None, True, False, 'auto'),
     doc="""Set up the dataset to be able to obtain content in the
     cheapest/fastest possible way, even if this poses a potential
     risk the data integrity (e.g. hardlink files from a local clone
diff --git a/datalad/interface/results.py b/datalad/interface/results.py
index 6324c9540..d6fd95830 100644
--- a/datalad/interface/results.py
+++ b/datalad/interface/results.py
@@ -262,6 +262,11 @@ def is_result_matching_pathsource_argument(res, **kwargs):
     # we either have any non-zero number of "paths" (that could be anything), or
     # we have one path and one source
     # we don't do any error checking here, done by the command itself
+    if res.get('action', None) not in ('install', 'get'):
+        # this filter is only used in install, reject anything that comes
+        # in that could not possibly be a 'install'-like result
+        # e.g. a sibling being added in the process
+        return False
     source = kwargs.get('source', None)
     if source is not None:
         # we want to be able to deal with Dataset instances given as 'source':
diff --git a/datalad/support/gitrepo.py b/datalad/support/gitrepo.py
index afc287c32..3af66829a 100644
--- a/datalad/support/gitrepo.py
+++ b/datalad/support/gitrepo.py
@@ -934,6 +934,18 @@ class GitRepo(RepoInterface, metaclass=PathBasedFlyweight):
                 gr.fsck()
             else:
                 lgr.warning("Experienced issues while cloning: %s", exc_str(fix_annex))
+        # make sure that Git doesn't mangle relative path specification into
+        # mildly obscure absolute paths
+        # https://github.com/datalad/datalad/issues/3538
+        if isinstance(url_ri, PathRI):
+            path = Path(url)
+            if not path.is_absolute():
+                # always in POSIX even on windows
+                path = path.as_posix()
+                if not path.startswith(op.pardir + '/'):
+                    path = posixpath.join(op.curdir, path)
+                gr.config.set('remote.origin.url', path,
+                              where='local', force=True)
         return gr
 
     def __del__(self):
diff --git a/datalad/utils.py b/datalad/utils.py
index 9eb7f1bc0..82a183659 100644
--- a/datalad/utils.py
+++ b/datalad/utils.py
@@ -1498,6 +1498,8 @@ class chpwd(object):
     def __init__(self, path, mkdir=False, logsuffix=''):
 
         if path:
+            # PY35 has no auto-conversion of Path to str
+            path = str(path)
             pwd = getpwd()
             self._prev_pwd = pwd
         else:


Test patch:
diff --git a/datalad/core/distributed/tests/__init__.py b/datalad/core/distributed/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/datalad/distribution/tests/test_clone.py b/datalad/core/distributed/tests/test_clone.py
similarity index 73%
rename from datalad/distribution/tests/test_clone.py
rename to datalad/core/distributed/tests/test_clone.py
index 02a6336d0..ae24f22b6 100644
--- a/datalad/distribution/tests/test_clone.py
+++ b/datalad/core/distributed/tests/test_clone.py
@@ -18,10 +18,6 @@ from datalad.tests.utils import (
 
 import logging
 import os
-from os import (
-    mkdir,
-    chmod,
-)
 import os.path as op
 
 from unittest.mock import patch
@@ -29,10 +25,11 @@ from unittest.mock import patch
 from datalad.api import (
     create,
     clone,
+    remove,
 )
 from datalad.utils import (
     chpwd,
-    _path_,
+    Path,
     on_windows,
 )
 from datalad.support.exceptions import IncompleteResultsError
@@ -56,17 +53,17 @@ from datalad.tests.utils import (
     assert_result_count,
     assert_result_values_equal,
     ok_startswith,
-    ok_clean_git,
+    assert_repo_status,
     serve_path_via_http,
     swallow_logs,
     use_cassette,
     skip_if_no_network,
     skip_if,
     with_sameas_remote,
+    known_failure,
     known_failure_appveyor,
-    known_failure_windows,
 )
-from datalad.distribution.clone import _get_installationpath_from_url
+from datalad.core.distributed.clone import _get_installationpath_from_url
 from datalad.distribution.dataset import Dataset
 
 
@@ -97,7 +94,7 @@ def test_invalid_args(path, otherpath, alienpath):
     ds = create(path)
     assert_raises(IncompleteResultsError, ds.clone, '/higherup.', 'Zoidberg')
     # make real dataset, try to install outside
-    ds_target = create(op.join(otherpath, 'target'))
+    ds_target = create(Path(otherpath) / 'target')
     assert_raises(ValueError, ds_target.clone, ds.path, path=ds.path)
     assert_status('error', ds_target.clone(ds.path, path=alienpath, on_failure='ignore'))
 
@@ -109,14 +106,15 @@ def test_invalid_args(path, otherpath, alienpath):
 @with_tempfile(mkdir=True)
 def test_clone_crcns(tdir, ds_path):
     with chpwd(tdir):
-        res = clone('///', path="all-nonrecursive", on_failure='ignore')
+        res = clone('///', path="all-nonrecursive", on_failure='ignore',
+                    result_xfm=None, return_type='list')
         assert_status('ok', res)
 
     # again, but into existing dataset:
     ds = create(ds_path)
     crcns = ds.clone("///crcns", result_xfm='datasets', return_type='item-or-list')
     ok_(crcns.is_installed())
-    eq_(crcns.path, op.join(ds_path, "crcns"))
+    eq_(crcns.pathobj, ds.pathobj / "crcns")
     assert_in(crcns.path, ds.subdatasets(result_xfm='paths'))
 
 
@@ -125,21 +123,21 @@ def test_clone_crcns(tdir, ds_path):
 @use_cassette('test_install_crcns')
 @with_tree(tree={'sub': {}})
 def test_clone_datasets_root(tdir):
+    tdir = Path(tdir)
     with chpwd(tdir):
-        ds = clone("///", result_xfm='datasets', return_type='item-or-list')
+        ds = clone("///")
         ok_(ds.is_installed())
-        eq_(ds.path, op.join(tdir, get_datasets_topdir()))
+        eq_(ds.pathobj, tdir / get_datasets_topdir())
 
         # do it a second time:
-        res = clone("///", on_failure='ignore')
+        res = clone("///", on_failure='ignore', result_xfm=None, return_type='list')
         assert_message(
             "dataset %s was already cloned from '%s'",
             res)
         assert_status('notneeded', res)
 
         # and a third time into an existing something, that is not a dataset:
-        with open(op.join(tdir, 'sub', 'a_file.txt'), 'w') as f:
-            f.write("something")
+        (tdir / 'sub' / 'a_file.txt').write_text("something")
 
         res = clone('///', path="sub", on_failure='ignore')
         assert_message(
@@ -148,8 +146,6 @@ def test_clone_datasets_root(tdir):
         assert_status('error', res)
 
 
-# https://github.com/datalad/datalad/pull/3975/checks?check_run_id=369789022#step:8:258
-@known_failure_windows
 @with_testrepos('.*basic.*', flavors=['local-url', 'network', 'local'])
 @with_tempfile(mkdir=True)
 def test_clone_simple_local(src, path):
@@ -169,21 +165,21 @@ def test_clone_simple_local(src, path):
         ok_(GitRepo.is_valid_repo(ds.path))
         eq_(set(ds.repo.get_indexed_files()),
             {'test.dat', 'INFO.txt'})
-        ok_clean_git(path, annex=False)
+        assert_repo_status(path, annex=False)
     else:
         # must be an annex
         ok_(isinstance(ds.repo, AnnexRepo))
         ok_(AnnexRepo.is_valid_repo(ds.path, allow_noninitialized=False))
         eq_(set(ds.repo.get_indexed_files()),
             {'test.dat', 'INFO.txt', 'test-annex.dat'})
-        ok_clean_git(path, annex=True)
+        assert_repo_status(path, annex=True)
         # no content was installed:
         ok_(not ds.repo.file_has_content('test-annex.dat'))
         uuid_before = ds.repo.uuid
         eq_(ds.repo.get_description(), 'mydummy')
 
     # installing it again, shouldn't matter:
-    res = clone(src, path)
+    res = clone(src, path, result_xfm=None, return_type='list')
     assert_result_values_equal(res, 'source_url', [src])
     assert_status('notneeded', res)
     assert_message("dataset %s was already cloned from '%s'", res)
@@ -192,8 +188,6 @@ def test_clone_simple_local(src, path):
         eq_(uuid_before, ds.repo.uuid)
 
 
-# https://github.com/datalad/datalad/pull/3975/checks?check_run_id=369789022#step:8:216
-@known_failure_windows
 @with_testrepos(flavors=['local-url', 'network', 'local'])
 @with_tempfile
 def test_clone_dataset_from_just_source(url, path):
@@ -203,10 +197,13 @@ def test_clone_dataset_from_just_source(url, path):
     ok_startswith(ds.path, path)
     ok_(ds.is_installed())
     ok_(GitRepo.is_valid_repo(ds.path))
-    ok_clean_git(ds.path, annex=None)
+    assert_repo_status(ds.path, annex=None)
     assert_in('INFO.txt', ds.repo.get_indexed_files())
 
 
+# test fails randomly, likely a bug in one of the employed test helpers
+# https://github.com/datalad/datalad/pull/3966#issuecomment-571267932
+@known_failure
 @with_tree(tree={
     'ds': {'test.txt': 'some'},
     })
@@ -214,7 +211,7 @@ def test_clone_dataset_from_just_source(url, path):
 @with_tempfile(mkdir=True)
 def test_clone_dataladri(src, topurl, path):
     # make plain git repo
-    ds_path = op.join(src, 'ds')
+    ds_path = Path(src) / 'ds'
     gr = GitRepo(ds_path, create=True)
     gr.add('test.txt')
     gr.commit('demo')
@@ -223,12 +220,10 @@ def test_clone_dataladri(src, topurl, path):
     with patch('datalad.consts.DATASETS_TOPURL', topurl):
         ds = clone('///ds', path, result_xfm='datasets', return_type='item-or-list')
     eq_(ds.path, path)
-    ok_clean_git(path, annex=False)
-    ok_file_has_content(op.join(path, 'test.txt'), 'some')
+    assert_repo_status(path, annex=False)
+    ok_file_has_content(ds.pathobj / 'test.txt', 'some')
 
 
-# https://github.com/datalad/datalad/pull/3975/checks?check_run_id=369789022#step:8:236
-@known_failure_windows
 @with_testrepos('submodule_annex', flavors=['local', 'local-url', 'network'])
 @with_tempfile(mkdir=True)
 @with_tempfile(mkdir=True)
@@ -243,7 +238,6 @@ def test_clone_isnot_recursive(src, path_nr, path_r):
         {'subm 1', '2'})
 
 
-
 @slow  # 23.1478s
 @with_testrepos(flavors=['local'])
 # 'local-url', 'network'
@@ -254,31 +248,29 @@ def test_clone_isnot_recursive(src, path_nr, path_r):
 def test_clone_into_dataset(source, top_path):
 
     ds = create(top_path)
-    ok_clean_git(ds.path)
+    assert_repo_status(ds.path)
 
     subds = ds.clone(source, "sub",
                      result_xfm='datasets', return_type='item-or-list')
-    ok_(op.isdir(op.join(subds.path, '.git')))
+    ok_((subds.pathobj / '.git').is_dir())
     ok_(subds.is_installed())
     assert_in('sub', ds.subdatasets(fulfilled=True, result_xfm='relpaths'))
     # sub is clean:
-    ok_clean_git(subds.path, annex=None)
+    assert_repo_status(subds.path, annex=None)
     # top is clean:
-    ok_clean_git(ds.path, annex=None)
+    assert_repo_status(ds.path, annex=None)
 
     # but we could also save while installing and there should be no side-effect
     # of saving any other changes if we state to not auto-save changes
     # Create a dummy change
     create_tree(ds.path, {'dummy.txt': 'buga'})
-    ok_clean_git(ds.path, untracked=['dummy.txt'])
+    assert_repo_status(ds.path, untracked=['dummy.txt'])
     subds_ = ds.clone(source, "sub2",
                       result_xfm='datasets', return_type='item-or-list')
-    eq_(subds_.path, op.join(ds.path, "sub2"))  # for paranoid yoh ;)
-    ok_clean_git(ds.path, untracked=['dummy.txt'])
+    eq_(subds_.pathobj, ds.pathobj / "sub2")  # for paranoid yoh ;)
+    assert_repo_status(ds.path, untracked=['dummy.txt'])
 
 
-# https://github.com/datalad/datalad/pull/3975/checks?check_run_id=369789022#step:8:278
-@known_failure_windows
 @with_testrepos('submodule_annex', flavors=['local', 'local-url', 'network'])
 @with_tempfile(mkdir=True)
 def test_notclone_known_subdataset(src, path):
@@ -287,7 +279,7 @@ def test_notclone_known_subdataset(src, path):
                result_xfm='datasets', return_type='item-or-list')
 
     # subdataset not installed:
-    subds = Dataset(op.join(path, 'subm 1'))
+    subds = Dataset(ds.pathobj / 'subm 1')
     assert_false(subds.is_installed())
     assert_in('subm 1', ds.subdatasets(fulfilled=False, result_xfm='relpaths'))
     assert_not_in('subm 1', ds.subdatasets(fulfilled=True, result_xfm='relpaths'))
@@ -331,9 +323,10 @@ def test_reckless(path, top_path):
 @with_tempfile
 @with_tempfile
 def test_install_source_relpath(src, dest):
+    src = Path(src)
     create(src)
-    src_ = op.basename(src)
-    with chpwd(op.dirname(src)):
+    src_ = src.name
+    with chpwd(src.parent):
         clone(src_, dest)
 
 
@@ -358,10 +351,10 @@ def test_clone_isnt_a_smartass(origin_path, path):
 @skip_if(on_windows or not os.geteuid(), "Will fail under super-user")
 @with_tempfile(mkdir=True)
 def test_clone_report_permission_issue(tdir):
-    pdir = _path_(tdir, 'protected')
-    mkdir(pdir)
+    pdir = Path(tdir) / 'protected'
+    pdir.mkdir()
     # make it read-only
-    chmod(pdir, 0o555)
+    pdir.chmod(0o555)
     with chpwd(pdir):
         res = clone('///', result_xfm=None, return_type='list', on_failure='ignore')
         assert_status('error', res)
@@ -381,7 +374,7 @@ def test_autoenabled_remote_msg(path):
     # whenever the remote we clone is the  type=git special remote, so the name
     # of the remote might not match
     with swallow_logs(new_level=logging.INFO) as cml:
-        res = clone('///repronim/containers', path)
+        res = clone('///repronim/containers', path, result_xfm=None, return_type='list')
         assert_status('ok', res)
         assert_not_in("not auto-enabled", cml.out)
 
@@ -404,15 +397,25 @@ def test_clone_autoenable_msg_handles_sameas(repo, clone_path):
 
 
 def test_installationpath_from_url():
-    for p in ('lastbit',
-              'lastbit/',
-              '/lastbit',
-              'lastbit.git',
-              'lastbit.git/',
-              'http://example.com/lastbit',
-              'http://example.com/lastbit.git',
-              'http://lastbit:8000'
-              ):
+    cases = (
+        'http://example.com/lastbit',
+        'http://example.com/lastbit.git',
+        'http://lastbit:8000',
+    ) + (
+        'C:\\Users\\mih\\AppData\\Local\\Temp\\lastbit',
+        'C:\\Users\\mih\\AppData\\Local\\Temp\\lastbit\\',
+        'Temp\\lastbit',
+        'Temp\\lastbit\\',
+        'lastbit.git',
+        'lastbit.git\\',
+    ) if on_windows else (
+        'lastbit',
+        'lastbit/',
+        '/lastbit',
+        'lastbit.git',
+        'lastbit.git/',
+    )
+    for p in cases:
         eq_(_get_installationpath_from_url(p), 'lastbit')
     # we need to deal with quoted urls
     for url in (
@@ -427,3 +430,72 @@ def test_installationpath_from_url():
     # and the hostname alone
     eq_(_get_installationpath_from_url("http://hostname"), 'hostname')
     eq_(_get_installationpath_from_url("http://hostname/"), 'hostname')
+
+
+# https://github.com/datalad/datalad/issues/3958
+@with_tempfile(mkdir=True)
+@with_tempfile(mkdir=True)
+def test_expanduser(srcpath, destpath):
+    src = Dataset(Path(srcpath) / 'src').create()
+    dest = Dataset(Path(destpath) / 'dest').create()
+
+    with chpwd(destpath), patch.dict('os.environ', {'HOME': srcpath}):
+        res = clone(op.join('~', 'src'), 'dest', result_xfm=None, return_type='list',
+                    on_failure='ignore')
+        assert_result_count(res, 1)
+        assert_result_count(
+            res, 1, action='install', status='error', path=dest.path,
+            message='target path already exists and not empty, refuse to '
+            'clone into target path')
+        # wipe out destination, and try again
+        assert_status('ok', remove(dataset=dest, check=False))
+        # now it should do it, and clone the right one
+        cloneds = clone(op.join('~', 'src'), 'dest')
+        eq_(cloneds.pathobj, Path(destpath) / 'dest')
+        eq_(src.id, cloneds.id)
+        # and it shouldn't fail when doing it again, because it detects
+        # the re-clone
+        cloneds = clone(op.join('~', 'src'), 'dest')
+        eq_(cloneds.pathobj, Path(destpath) / 'dest')
+
+
+@with_tempfile(mkdir=True)
+def test_cfg_originorigin(path):
+    path = Path(path)
+    origin = Dataset(path / 'origin').create()
+    (origin.pathobj / 'file1.txt').write_text('content')
+    origin.save()
+    clone_direct = clone(origin, path / 'clone_direct')
+    clone_clone = clone(clone_direct, path / 'clone_clone')
+    # the goal is to be able to get file content from origin without
+    # the need to configure it manually
+    assert_result_count(
+        clone_clone.get('file1.txt', on_failure='ignore'),
+        1,
+        action='get',
+        status='ok',
+        path=str(clone_clone.pathobj / 'file1.txt'),
+    )
+    eq_((clone_clone.pathobj / 'file1.txt').read_text(), 'content')
+    eq_(
+        Path(clone_clone.siblings(
+            'query',
+            name='origin-2',
+            return_type='item-or-list')['url']),
+        origin.pathobj
+    )
+
+
+# test fix for gh-2601/gh-3538
+@with_tempfile()
+def test_relative_submodule_url(path):
+    Dataset(op.join(path, 'origin')).create()
+    ds = Dataset(op.join(path, 'ds')).create()
+    with chpwd(ds.path):
+        ds.clone(
+            source=op.join(op.pardir, 'origin'),
+            path='sources')
+    subinfo = ds.subdatasets(return_type='item-or-list')
+    eq_(subinfo['gitmodule_url'],
+        # must be a relative URL, not platform-specific relpath!
+        '{}/{}'.format(op.pardir, 'origin'))
diff --git a/datalad/distribution/tests/test_get.py b/datalad/distribution/tests/test_get.py
index 98b6fa31e..4ea83ccba 100644
--- a/datalad/distribution/tests/test_get.py
+++ b/datalad/distribution/tests/test_get.py
@@ -140,13 +140,13 @@ def test_get_flexible_source_candidates_for_submodule(t, t2, t3):
     clone3.siblings('add', name='myremote', url=ds.path,
                     result_renderer='disabled')
     clone3.update(sibling='myremote')
-    # we should end up with three pieces
-    eq_(f(clone3, clone3.subdatasets(return_type='item-or-list')),
-        [
-            ('origin', clone_subpath),
-            ('myremote', ds_subpath),
-            ('local', str(clone3.pathobj / 'sub')),
-    ])
+    # we should end up with this additional piece
+    # we are not checking for the name of the remote, because it is actually
+    # registered under two different names
+    assert_in(
+        ds_subpath,
+        [i[1] for i in f(clone3, clone3.subdatasets(return_type='item-or-list'))]
+    )
 
     # TODO: check that http:// urls for the dataset itself get resolved
     # TODO: many more!!
diff --git a/datalad/distribution/tests/test_install.py b/datalad/distribution/tests/test_install.py
index ed14937c6..9008f59f9 100644
--- a/datalad/distribution/tests/test_install.py
+++ b/datalad/distribution/tests/test_install.py
@@ -340,21 +340,21 @@ def test_install_recursive(src, path_nr, path_r):
 
     # now recursively:
     # don't filter implicit results so we can inspect them
-    ds_list = install(path_r, source=src, recursive=True, result_filter=None)
+    res = install(path_r, source=src, recursive=True,
+                  result_xfm=None, result_filter=None)
     # installed a dataset and two subdatasets
-    eq_(len(ds_list), 3)
-    eq_(sum([isinstance(i, Dataset) for i in ds_list]), 3)
+    assert_result_count(res, 3, action='install', type='dataset')
     # we recurse top down during installation, so toplevel should appear at
     # first position in returned list
-    eq_(ds_list[0].path, path_r)
-    top_ds = ds_list[0]
+    eq_(res[0]['path'], path_r)
+    top_ds = Dataset(res[0]['path'])
     ok_(top_ds.is_installed())
 
     # the subdatasets are contained in returned list:
     # (Note: Until we provide proper (singleton) instances for Datasets,
     # need to check for their paths)
-    assert_in(opj(top_ds.path, 'subm 1'), [i.path for i in ds_list])
-    assert_in(opj(top_ds.path, '2'), [i.path for i in ds_list])
+    assert_in_results(res, path=opj(top_ds.path, 'subm 1'), type='dataset')
+    assert_in_results(res, path=opj(top_ds.path, '2'), type='dataset')
 
     eq_(len(top_ds.subdatasets(recursive=True)), 2)
 
@@ -389,8 +389,8 @@ def test_install_recursive_with_data(src, path):
     assert_status('ok', res)
     # installed a dataset and two subdatasets, and one file with content in
     # each
-    eq_(len(res), 6)
-    assert_result_count(res, 3, type='dataset')
+    assert_result_count(res, 3, type='dataset', action='install')
+    assert_result_count(res, 3, type='file', action='get')
     # we recurse top down during installation, so toplevel should appear at
     # first position in returned list
     eq_(res[0]['path'], path)
diff --git a/datalad/distribution/tests/test_siblings.py b/datalad/distribution/tests/test_siblings.py
index 3971d063f..865607649 100644
--- a/datalad/distribution/tests/test_siblings.py
+++ b/datalad/distribution/tests/test_siblings.py
@@ -311,8 +311,7 @@ def test_sibling_enable_sameas(repo, clone_path):
     ds.repo.copy_to(["f0"], remote="r_dir")
     ds.repo.drop(["f0"])
 
-    ds_cloned = clone(ds.path, clone_path,
-                      result_xfm="datasets", return_type="item-or-list")
+    ds_cloned = clone(ds.path, clone_path)
 
     assert_false(ds_cloned.repo.file_has_content("f0"))
     res = ds_cloned.siblings(action="enable", name="r_rsync")
@@ -331,7 +330,7 @@ def test_sibling_inherit(basedir):
                       annex_group="grp", result_renderer=None)
 
     ds_clone = ds_super.clone(
-        source=ds_source.path, path="clone", result_xfm="datasets")[0]
+        source=ds_source.path, path="clone")
     # In a subdataset, adding a "source" sibling with inherit=True pulls in
     # that configuration.
     ds_clone.siblings(action="add", name="source", url=ds_source.path,
@@ -346,7 +345,7 @@ def test_sibling_inherit_no_super_remote(basedir):
     ds_source = Dataset(opj(basedir, "source")).create()
     ds_super = Dataset(opj(basedir, "super")).create()
     ds_clone = ds_super.clone(
-        source=ds_source.path, path="clone", result_xfm="datasets")[0]
+        source=ds_source.path, path="clone")
     # Adding a sibling with inherit=True doesn't crash when the superdataset
     # doesn't have a remote `name`.
     ds_clone.siblings(action="add", name="donotexist", inherit=True,
diff --git a/datalad/support/tests/test_network.py b/datalad/support/tests/test_network.py
index 360030257..90d2f400a 100644
--- a/datalad/support/tests/test_network.py
+++ b/datalad/support/tests/test_network.py
@@ -481,7 +481,8 @@ def test_get_local_file_url_compatibility(path):
     # compat with git clone/submodule
     assert_status(
         'ok',
-        ds1.clone(get_local_file_url(ds2.path, compatibility='git')))
+        ds1.clone(get_local_file_url(ds2.path, compatibility='git'),
+                  result_xfm=None, return_type='generator'))
 
 
 def test_is_ssh():
diff --git a/datalad/tests/utils.py b/datalad/tests/utils.py
index 51b4833b8..b6865043b 100644
--- a/datalad/tests/utils.py
+++ b/datalad/tests/utils.py
@@ -47,10 +47,15 @@ from nose.tools import assert_is_instance
 from nose import SkipTest
 
 import datalad.utils as ut
+# TODO this must go
+from ..utils import *
+from datalad.utils import (
+    Path,
+    ensure_unicode,
+)
 
 from ..cmd import Runner
 from .. import utils
-from ..utils import *
 from ..support.exceptions import CommandNotAvailableError
 from ..support.vcr_ import *
 from ..support.keyring_ import MemoryKeyring
@@ -240,7 +245,6 @@ def skip_v6_or_later(func, method='raise'):
 
 import git
 import os
-from os.path import exists, join
 from datalad.support.gitrepo import GitRepo
 from datalad.support.annexrepo import AnnexRepo, FileNotInAnnexError
 from datalad.distribution.dataset import Dataset
@@ -501,26 +505,27 @@ def ok_archives_caches(repopath, n=1, persistent=None):
 
 
 def ok_exists(path):
-    assert exists(path), 'path %s does not exist (or dangling symlink)' % path
+    assert Path(path).exists(), 'path %s does not exist (or dangling symlink)' % path
 
 
 def ok_file_has_content(path, content, strip=False, re_=False,
                         decompress=False, **kwargs):
     """Verify that file exists and has expected content"""
+    path = Path(path)
     ok_exists(path)
     if decompress:
-        if path.endswith('.gz'):
+        if path.suffix == '.gz':
             open_func = gzip.open
         else:
             raise NotImplementedError("Don't know how to decompress %s" % path)
     else:
         open_func = open
 
-    with open_func(path, 'rb') as f:
+    with open_func(str(path), 'rb') as f:
         file_content = f.read()
 
     if isinstance(content, str):
-        file_content = assure_unicode(file_content)
+        file_content = ensure_unicode(file_content)
 
     if os.linesep != '\n':
         # for consistent comparisons etc. Apparently when reading in `b` mode
diff --git a/datalad/tests/utils_testdatasets.py b/datalad/tests/utils_testdatasets.py
index 36cec53ee..2adf13d88 100644
--- a/datalad/tests/utils_testdatasets.py
+++ b/datalad/tests/utils_testdatasets.py
@@ -30,27 +30,27 @@ def make_studyforrest_mockup(path):
                            description='old-style, no connection to RAW')
     structural = public.create('structural', description='anatomy')
     tnt = public.create('tnt', description='image templates')
-    tnt.clone(source=phase1.path, path=opj('src', 'phase1'), reckless=True)
-    tnt.clone(source=structural.path, path=opj('src', 'structural'), reckless=True)
+    tnt.clone(source=phase1.path, path=opj('src', 'phase1'), reckless='auto')
+    tnt.clone(source=structural.path, path=opj('src', 'structural'), reckless='auto')
     aligned = public.create('aligned', description='aligned image data')
-    aligned.clone(source=phase1.path, path=opj('src', 'phase1'), reckless=True)
-    aligned.clone(source=tnt.path, path=opj('src', 'tnt'), reckless=True)
+    aligned.clone(source=phase1.path, path=opj('src', 'phase1'), reckless='auto')
+    aligned.clone(source=tnt.path, path=opj('src', 'tnt'), reckless='auto')
     # new acquisition
     labet = create(opj(path, 'private', 'labet'), description="raw data ET")
     phase2_dicoms = create(opj(path, 'private', 'p2dicoms'), description="raw data P2MRI")
     phase2 = public.create('phase2',
                            description='new-style, RAW connection')
-    phase2.clone(source=labet.path, path=opj('src', 'labet'), reckless=True)
-    phase2.clone(source=phase2_dicoms.path, path=opj('src', 'dicoms'), reckless=True)
+    phase2.clone(source=labet.path, path=opj('src', 'labet'), reckless='auto')
+    phase2.clone(source=phase2_dicoms.path, path=opj('src', 'dicoms'), reckless='auto')
     # add to derivatives
-    tnt.clone(source=phase2.path, path=opj('src', 'phase2'), reckless=True)
-    aligned.clone(source=phase2.path, path=opj('src', 'phase2'), reckless=True)
+    tnt.clone(source=phase2.path, path=opj('src', 'phase2'), reckless='auto')
+    aligned.clone(source=phase2.path, path=opj('src', 'phase2'), reckless='auto')
     # never to be published media files
     media = create(opj(path, 'private', 'media'), description="raw data ET")
     # assuming all annotations are in one dataset (in reality this is also
     # a superdatasets with about 10 subdatasets
     annot = public.create('annotations', description='stimulus annotation')
-    annot.clone(source=media.path, path=opj('src', 'media'), reckless=True)
+    annot.clone(source=media.path, path=opj('src', 'media'), reckless='auto')
     # a few typical analysis datasets
     # (just doing 3, actual status quo is just shy of 10)
     # and also the real goal -> meta analysis
@@ -58,12 +58,12 @@ def make_studyforrest_mockup(path):
     for i in range(1, 3):
         ana = public.create('analysis{}'.format(i),
                             description='analysis{}'.format(i))
-        ana.clone(source=annot.path, path=opj('src', 'annot'), reckless=True)
-        ana.clone(source=aligned.path, path=opj('src', 'aligned'), reckless=True)
-        ana.clone(source=tnt.path, path=opj('src', 'tnt'), reckless=True)
+        ana.clone(source=annot.path, path=opj('src', 'annot'), reckless='auto')
+        ana.clone(source=aligned.path, path=opj('src', 'aligned'), reckless='auto')
+        ana.clone(source=tnt.path, path=opj('src', 'tnt'), reckless='auto')
         # link to metaanalysis
         metaanalysis.clone(source=ana.path, path=opj('src', 'ana{}'.format(i)),
-                           reckless=True)
+                           reckless='auto')
         # simulate change in an input (but not raw) dataset
         create_tree(
             aligned.path,
@@ -71,7 +71,7 @@ def make_studyforrest_mockup(path):
         aligned.save()
     # finally aggregate data
     aggregate = public.create('aggregate', description='aggregate data')
-    aggregate.clone(source=aligned.path, path=opj('src', 'aligned'), reckless=True)
+    aggregate.clone(source=aligned.path, path=opj('src', 'aligned'), reckless='auto')
     # the toplevel dataset is intentionally left dirty, to reflect the
     # most likely condition for the joint dataset to be in at any given
     # point in time