Repository: datalad/datalad

Issue:
S3 credential issue not communicated and breaks special remote protocol in master
This is similar to #5469, but a different issue.

I have (intentionally) broken S3 credentials in my keyring. Now trying to download a file from the HCP dataset

```
% datalad get HCP1200/143426/T1w/T1w_acpc_dc.nii.gz
get(error): HCP1200/143426/T1w/T1w_acpc_dc.nii.gz (file) [TRANSFER failed with no reason given                             
TRANSFER failed with no reason given
TRANSFER failed with no reason given]
action summary:
  get (error: 1, notneeded: 1)
```

Because I created the invalid credentials, I no what the reason is. However, in the general case it seems unresonable to not be able to tell that a download failed because of invalid authentication. Even turning on `-l debug` does not reveal more information.

Comments:

Patch:
diff --git a/datalad/customremotes/__init__.py b/datalad/customremotes/__init__.py
index 81ee915b8..a35852f75 100644
--- a/datalad/customremotes/__init__.py
+++ b/datalad/customremotes/__init__.py
@@ -11,3 +11,62 @@
 """
 
 __docformat__ = 'restructuredtext'
+
+__all__ = ['RemoteError', 'SpecialRemote']
+
+from annexremote import (
+    ProtocolError,
+    SpecialRemote as _SpecialRemote,
+    RemoteError as _RemoteError,
+)
+from datalad.support.exceptions import format_exception_with_cause
+
+
+class RemoteError(_RemoteError):
+    def __str__(self):
+        # this is a message given to remote error, if any
+        exc_str = super().__str__()
+        # this is the cause ala `raise from`
+        exc_cause = getattr(self, '__cause__', None)
+        if exc_cause:
+            # if we have a cause, collect the cause all the way down
+            # we can do quite some chaining
+            exc_cause = format_exception_with_cause(exc_cause)
+        if exc_str and exc_cause:
+            # with have the full picture
+            msg = f'{exc_str} caused by {exc_cause}'
+        elif exc_str and not exc_cause:
+            # only a custom message
+            msg = exc_str
+        elif not exc_str and exc_cause:
+            # only the cause
+            msg = exc_cause
+        else:
+            # nothing, shame!
+            msg = 'exception with unknown cause'
+        # prevent multiline messages, they would be swallowed
+        # or kill the protocol
+        return msg.replace('\n', '\\n')
+
+
+class SpecialRemote(_SpecialRemote):
+    """Common base class for all of DataLad's special remote implementations"""
+
+    def message(self, msg, type='debug'):
+        handler = dict(
+            debug=self.annex.debug,
+            info=self.annex.info,
+            error=self.annex.error,
+        ).get(type, self.annex.debug)
+
+        # ensure that no multiline messages are sent, they would cause a
+        # protocol error
+        msg = msg.replace('\n', '\\n')
+
+        try:
+            handler(msg)
+        except ProtocolError:
+            # INFO not supported by annex version.
+            # If we can't have an actual info message, at least have a
+            # debug message.
+            self.annex.debug(msg)
diff --git a/datalad/customremotes/archives.py b/datalad/customremotes/archives.py
index c9613c04d..a17ad5854 100644
--- a/datalad/customremotes/archives.py
+++ b/datalad/customremotes/archives.py
@@ -18,10 +18,7 @@ from collections import OrderedDict
 from operator import itemgetter
 from urllib.parse import urlparse
 
-from annexremote import (
-    RemoteError,
-    UnsupportedRequest,
-)
+from annexremote import UnsupportedRequest
 
 from datalad.cmdline.helpers import get_repo_instance
 from datalad.consts import ARCHIVES_SPECIAL_REMOTE
@@ -38,6 +35,7 @@ from datalad.utils import (
     unlink,
 )
 
+from datalad.customremotes import RemoteError
 from .base import AnnexCustomRemote
 
 lgr = logging.getLogger('datalad.customremotes.archive')
@@ -329,7 +327,7 @@ class ArchiveAnnexCustomRemote(AnnexCustomRemote):
                 return True
         # it is unclear to MIH why this must be UNKNOWN rather than FALSE
         # but this is how I found it
-        raise RemoteError()
+        raise RemoteError('Key not present')
 
     def remove(self, key):
         raise UnsupportedRequest('This special remote cannot remove content')
@@ -410,17 +408,18 @@ class ArchiveAnnexCustomRemote(AnnexCustomRemote):
                 apath = self.cache[akey_path].get_extracted_file(afile)
                 link_file_load(apath, file)
                 if not was_extracted and self.cache[akey_path].is_extracted:
-                    self.annex.info(
+                    self.message(
                         "%s special remote is using an extraction cache "
                         "under %s. Remove it with DataLad's 'clean' "
                         "command to save disk space." %
                         (ARCHIVES_SPECIAL_REMOTE,
-                         self.cache[akey_path].path)
+                         self.cache[akey_path].path),
+                        type='info',
                     )
                 return
             except Exception as exc:
                 ce = CapturedException(exc)
-                self.annex.debug(
+                self.message(
                     "Failed to fetch {akey} containing {key}: {msg}".format(
                         akey=akey,
                         key=key,
@@ -453,10 +452,11 @@ class ArchiveAnnexCustomRemote(AnnexCustomRemote):
         from datalad.support.annexrepo import AnnexJsonProtocol
 
         akey_size = self.repo.get_size_from_key(akey)
-        self.annex.info(
+        self.message(
             "To obtain some keys we need to fetch an archive "
             "of size %s"
-            % (naturalsize(akey_size) if akey_size else "unknown")
+            % (naturalsize(akey_size) if akey_size else "unknown"),
+            type='info',
         )
 
         try:
@@ -465,7 +465,7 @@ class ArchiveAnnexCustomRemote(AnnexCustomRemote):
                 protocol=AnnexJsonProtocol,
             )
         except Exception:
-            self.annex.debug(f'Failed to fetch archive with key {akey}')
+            self.message(f'Failed to fetch archive with key {akey}')
             raise
 
 
diff --git a/datalad/customremotes/base.py b/datalad/customremotes/base.py
index 3df0045e5..1dc1a6aac 100644
--- a/datalad/customremotes/base.py
+++ b/datalad/customremotes/base.py
@@ -20,9 +20,9 @@ lgr = logging.getLogger('datalad.customremotes')
 
 from annexremote import (
     RemoteError,
-    SpecialRemote,
     UnsupportedRequest,
 )
+from datalad.customremotes import SpecialRemote
 
 from datalad.ui import ui
 
diff --git a/datalad/customremotes/datalad.py b/datalad/customremotes/datalad.py
index 2bc639f81..b8795d317 100644
--- a/datalad/customremotes/datalad.py
+++ b/datalad/customremotes/datalad.py
@@ -13,15 +13,15 @@ __docformat__ = 'restructuredtext'
 import logging
 from urllib.parse import urlparse
 
-from annexremote import RemoteError
-
 from datalad.downloaders.providers import Providers
 from datalad.support.exceptions import (
     CapturedException,
     TargetFileAbsent,
 )
+from datalad.utils import unique
 
-from .base import AnnexCustomRemote
+from datalad.customremotes import RemoteError
+from datalad.customremotes.base import AnnexCustomRemote
 
 lgr = logging.getLogger('datalad.customremotes.datalad')
 
@@ -39,6 +39,7 @@ class DataladAnnexCustomRemote(AnnexCustomRemote):
 
     def transfer_retrieve(self, key, file):
         urls = []
+        error_causes = []
         # TODO: priorities etc depending on previous experience or settings
         for url in self.gen_URLS(key):
             urls.append(url)
@@ -50,10 +51,17 @@ class DataladAnnexCustomRemote(AnnexCustomRemote):
                 return
             except Exception as exc:
                 ce = CapturedException(exc)
-                self.annex.debug("Failed to download url %s for key %s: %s"
-                                 % (url, key, ce))
-        raise RemoteError(
-            f"Failed to download from any of {len(urls)} locations")
+                cause = getattr(exc, '__cause__', None)
+                debug_msg = f"Failed to download {url} for key {key}: {ce}"
+                if cause:
+                    debug_msg += f' [{cause}]'
+                self.message(debug_msg)
+                error_causes.append(cause)
+
+        error_msg = f"Failed to download from any of {len(urls)} locations"
+        if error_causes:
+            error_msg += f' {unique(error_causes)}'
+        raise RemoteError(error_msg)
 
     def checkurl(self, url):
         try:
@@ -64,7 +72,7 @@ class DataladAnnexCustomRemote(AnnexCustomRemote):
             return [props]
         except Exception as exc:
             ce = CapturedException(exc)
-            self.annex.debug("Failed to check url %s: %s" % (url, ce))
+            self.message("Failed to check url %s: %s" % (url, ce))
             return False
 
     def checkpresent(self, key):
@@ -80,7 +88,7 @@ class DataladAnnexCustomRemote(AnnexCustomRemote):
                 # N/A, probably check the connection etc
             except TargetFileAbsent as exc:
                 ce = CapturedException(exc)
-                self.annex.debug(
+                self.message(
                     "Target url %s file seems to be missing: %s" % (url, ce))
                 if not resp:
                     # if it is already marked as UNKNOWN -- let it stay that
@@ -89,7 +97,7 @@ class DataladAnnexCustomRemote(AnnexCustomRemote):
                     return False
             except Exception as exc:
                 ce = CapturedException(exc)
-                self.annex.debug(
+                self.message(
                     "Failed to check status of url %s: %s" % (url, ce))
         if resp is None:
             raise RemoteError(f'Could not determine presence of key {key}')
diff --git a/datalad/distributed/ora_remote.py b/datalad/distributed/ora_remote.py
index 85bf1777b..282f2ab3e 100644
--- a/datalad/distributed/ora_remote.py
+++ b/datalad/distributed/ora_remote.py
@@ -1,7 +1,3 @@
-from annexremote import SpecialRemote
-from annexremote import RemoteError
-from annexremote import ProtocolError
-
 import os
 from pathlib import (
     Path,
@@ -13,6 +9,11 @@ from shlex import quote as sh_quote
 import subprocess
 import logging
 from functools import wraps
+
+from datalad.customremotes import (
+    RemoteError,
+    SpecialRemote,
+)
 from datalad.customremotes.ria_utils import (
     get_layout_locations,
     UnknownLayoutVersion,
@@ -78,9 +79,7 @@ class RemoteCommandFailedError(Exception):
 
 
 class RIARemoteError(RemoteError):
-
-    def __init__(self, msg):
-        super().__init__(msg.replace('\n', '\\n'))
+    pass
 
 
 class IOBase(object):
@@ -214,9 +213,9 @@ class LocalIO(IOBase):
         try:
             path.unlink()
         except PermissionError as e:
-            raise RIARemoteError(str(e) + os.linesep +
-                                 "Note: Write permissions for a key's parent"
-                                 "directory are also required to drop content.")
+            raise RIARemoteError(
+                "Write permissions for a key's parent directory are "
+                "also required to drop content.") from e
 
     def remove_dir(self, path):
         path.rmdir()
@@ -472,7 +471,7 @@ class SSHRemoteIO(IOBase):
         try:
             size = self._get_download_size_from_key(key)
         except RemoteError as e:
-            raise RemoteError("src: {}".format(str(src)) + str(e))
+            raise RemoteError(f"src: {src}") from e
 
         if size is None:
             # rely on SCP for now
@@ -501,9 +500,8 @@ class SSHRemoteIO(IOBase):
             self._run('rm {}'.format(sh_quote(str(path))))
         except RemoteCommandFailedError as e:
             raise RIARemoteError(
-                str(e) + os.linesep +
-                "Note: Write permissions for a key's parent"
-                "directory are also required to drop content.")
+                "Write permissions for a key's parent"
+                "directory are also required to drop content.") from e
 
     def remove_dir(self, path):
         self._run('rmdir {}'.format(sh_quote(str(path))))
@@ -575,8 +573,8 @@ class SSHRemoteIO(IOBase):
         cmd = "cat  {}".format(sh_quote(str(file_path)))
         try:
             out = self._run(cmd, no_output=False, check=True)
-        except RemoteCommandFailedError:
-            raise RIARemoteError("Could not read {}".format(str(file_path)))
+        except RemoteCommandFailedError as e:
+            raise RIARemoteError(f"Could not read {file_path}") from e
 
         return out
 
@@ -597,8 +595,8 @@ class SSHRemoteIO(IOBase):
             sh_quote(str(file_path)))
         try:
             self._run(cmd, check=True)
-        except RemoteCommandFailedError:
-            raise RIARemoteError("Could not write to {}".format(str(file_path)))
+        except RemoteCommandFailedError as e:
+            raise RIARemoteError(f"Could not write to {file_path}") from e
 
     def get_7z(self):
         # TODO: To not rely on availability in PATH we might want to use `which`
@@ -666,7 +664,7 @@ class HTTPRemoteIO(object):
         try:
             response = requests.head(url, allow_redirects=True)
         except Exception as e:
-            raise RIARemoteError(str(e))
+            raise RIARemoteError from e
 
         return response.status_code == 200
 
@@ -736,7 +734,7 @@ def handle_errors(func):
                 pass
 
             if not isinstance(e, RIARemoteError):
-                raise RIARemoteError(str(e))
+                raise RIARemoteError from e
             else:
                 raise e
 
@@ -874,7 +872,7 @@ class RIARemote(SpecialRemote):
                 self._get_version_config(object_tree_version_file)
             if self.remote_object_tree_version not in self.known_versions_objt:
                 raise UnknownLayoutVersion
-        except (RemoteError, FileNotFoundError):
+        except (RemoteError, FileNotFoundError) as e:
             # Exception class depends on whether self.io is local or SSH.
             # assume file doesn't exist
             # TODO: Is there a possibility RemoteError has a different reason
@@ -882,9 +880,9 @@ class RIARemote(SpecialRemote):
             #       Don't think so ATM. -> Reconsider with new execution layer.
             if not self.io.exists(object_tree_version_file.parent):
                 # unify exception
-                raise FileNotFoundError
+                raise e
             else:
-                raise NoLayoutVersion
+                raise NoLayoutVersion from e
 
     def _load_cfg(self, gitdir, name):
         # Whether or not to force writing to the remote. Currently used to
@@ -953,7 +951,8 @@ class RIARemote(SpecialRemote):
                              " Use 'git annex enableremote {} "
                              "url=<RIA-URL-TO-STORE>' to store a ria+<scheme>:"
                              "//... URL in the special remote's config."
-                             "".format(name))
+                             "".format(name),
+                             type='info')
 
         if not self.store_base_path:
             raise RIARemoteError(
@@ -997,7 +996,8 @@ class RIARemote(SpecialRemote):
 
         file_content = self.io.read_file(path).strip().split('|')
         if not (1 <= len(file_content) <= 2):
-            self.message("invalid version file {}".format(path))
+            self.message("invalid version file {}".format(path),
+                         type='info')
             return None
 
         remote_version = file_content[0]
@@ -1097,27 +1097,13 @@ class RIARemote(SpecialRemote):
         #       + just isinstance(LocalIO)?
         return not self.storage_host
 
-    def debug(self, msg):
-        # Annex prints just the message, so prepend with
-        # a "DEBUG" on our own.
-        self.annex.debug("ORA-DEBUG: " + msg)
-
-    def message(self, msg):
-        try:
-            self.annex.info(msg)
-        except ProtocolError:
-            # INFO not supported by annex version.
-            # If we can't have an actual info message, at least have a
-            # debug message.
-            self.debug(msg)
-
     def _set_read_only(self, msg):
 
         if not self.force_write:
             self.read_only = True
-            self.message(msg)
+            self.message(msg, type='info')
         else:
-            self.message("Was instructed to force write")
+            self.message("Was instructed to force write", type='info')
 
     def _ensure_writeable(self):
         if self.read_only:
@@ -1169,7 +1155,7 @@ class RIARemote(SpecialRemote):
 
         if not self._push_io:
             if self.ria_store_pushurl:
-                self.debug("switching ORA to push-url")
+                self.message("switching ORA to push-url")
                 # Not-implemented-push-HTTP is ruled out already when reading
                 # push-url, so either local or SSH:
                 if not self.storage_host_push:
@@ -1291,7 +1277,8 @@ class RIARemote(SpecialRemote):
                 self.io.get_from_archive(archive_path, key_path, filename,
                                          self.annex.progress)
             except Exception as e2:
-                raise RIARemoteError('Failed to key: {}'
+                # TODO properly report the causes
+                raise RIARemoteError('Failed to obtain key: {}'
                                      ''.format([str(e1), str(e2)]))
 
     @handle_errors
diff --git a/datalad/downloaders/s3.py b/datalad/downloaders/s3.py
index 48260d547..e0dc5f4dc 100644
--- a/datalad/downloaders/s3.py
+++ b/datalad/downloaders/s3.py
@@ -351,8 +351,8 @@ class S3Downloader(BaseDownloader):
                 url_filepath, version_id=params.get('versionId', None)
             )
         except S3ResponseError as e:
-            raise TargetFileAbsent("S3 refused to provide the key for %s from url %s: %s"
-                                % (url_filepath, url, e))
+            raise TargetFileAbsent("S3 refused to provide the key for %s from url %s"
+                                % (url_filepath, url)) from e
         if key is None:
             raise TargetFileAbsent("No key returned for %s from url %s" % (url_filepath, url))
 
diff --git a/datalad/support/exceptions.py b/datalad/support/exceptions.py
index 9cd5d11a6..00867a803 100644
--- a/datalad/support/exceptions.py
+++ b/datalad/support/exceptions.py
@@ -185,6 +185,26 @@ def format_oneline_tb(exc, tb=None, limit=None, include_str=True):
     return out
 
 
+def format_exception_with_cause(e):
+    """Helper to recursively format an exception with all underlying causes
+
+    For each exception in the chain either the str() of it is taken, or the
+    class name of the exception, with the aim to generate a simple and
+    comprehensible description that can be used in user-facing messages.
+    It is explicitly not aiming to provide a detailed/comprehensive source
+    of information for in-depth debugging.
+
+    '-caused by-' is used a separator between exceptions to be human-readable
+    while being recognizably different from potential exception payload
+    messages.
+    """
+    s = str(e) or e.__class__.__name__
+    exc_cause = getattr(e, '__cause__', None)
+    if exc_cause:
+        s += f' -caused by- {format_exception_with_cause(exc_cause)}'
+    return s
+
+
 class MissingExternalDependency(RuntimeError):
     """External dependency is missing error"""
 


Test patch:
diff --git a/datalad/support/tests/test_captured_exception.py b/datalad/support/tests/test_captured_exception.py
index 4d56fa262..c8c22b77d 100644
--- a/datalad/support/tests/test_captured_exception.py
+++ b/datalad/support/tests/test_captured_exception.py
@@ -1,7 +1,13 @@
 from unittest.mock import patch
-from nose.tools import assert_equal, assert_true
-from datalad.support.exceptions import CapturedException
-from datalad.tests.utils import assert_re_in
+from datalad.support.exceptions import (
+    format_exception_with_cause,
+    CapturedException,
+)
+from datalad.tests.utils import (
+    assert_equal,
+    assert_re_in,
+    assert_true,
+)
 from datalad import cfg
 
 
@@ -78,3 +84,28 @@ def test_CapturedException():
     # CapturedException.__repr__:
     assert_re_in(r".*test_captured_exception.py:f2:[0-9]+\]$",
                  captured_exc.__repr__())
+
+
+def makeitraise():
+    def raise_valueerror():
+        try:
+            raise_runtimeerror()
+        except Exception as e:
+            raise ValueError from e
+
+    def raise_runtimeerror():
+        raise RuntimeError("Mike")
+
+    try:
+        raise_valueerror()
+    except Exception as e:
+        raise RuntimeError from e
+
+
+def test_format_exception_with_cause():
+    try:
+        makeitraise()
+    except Exception as e:
+        assert_equal(
+            format_exception_with_cause(e),
+            'RuntimeError -caused by- ValueError -caused by- Mike')