Repository: datalad/datalad

Issue:
annexjson2result cannot handle file=null results
Those are produced by `git annex export` -- although they likely should not be happening.

```
  File "/home/mih/hacking/datalad/git/datalad/interface/results.py", line 228, in annexjson2result
    res['path'] = str(ds.pathobj / PurePosixPath(d['file']))
  File "/usr/lib/python3.9/pathlib.py", line 664, in __new__
    return cls._from_parts(args)
  File "/usr/lib/python3.9/pathlib.py", line 696, in _from_parts
    drv, root, parts = self._parse_args(args)
  File "/usr/lib/python3.9/pathlib.py", line 680, in _parse_args
    a = os.fspath(a)
TypeError: expected str, bytes or os.PathLike object, not NoneType
```

Related: https://git-annex.branchable.com/bugs/JSON_results_for___96__export__96___have_file__58__null_property

Comments:

Patch:
diff --git a/datalad/interface/__init__.py b/datalad/interface/__init__.py
index f5a4c5b55..5ac873a5d 100644
--- a/datalad/interface/__init__.py
+++ b/datalad/interface/__init__.py
@@ -73,6 +73,7 @@ _group_misc = (
         ('datalad.interface.add_archive_content', 'AddArchiveContent',
          'add-archive-content'),
         ('datalad.interface.download_url', 'DownloadURL', 'download-url'),
+        ('datalad.interface.shell_completion', 'ShellCompletion', 'shell-completion'),
         ('datalad.core.local.run', 'Run', 'run'),
         ('datalad.interface.rerun', 'Rerun', 'rerun'),
         ('datalad.interface.run_procedure', 'RunProcedure', 'run-procedure'),
diff --git a/datalad/interface/results.py b/datalad/interface/results.py
index ae98b228a..4769d1a46 100644
--- a/datalad/interface/results.py
+++ b/datalad/interface/results.py
@@ -224,7 +224,7 @@ def annexjson2result(d, ds, **kwargs):
     res['status'] = 'ok' if d.get('success', False) is True else 'error'
     # we cannot rely on any of these to be available as the feed from
     # git annex (or its wrapper) is not always homogeneous
-    if 'file' in d:
+    if d.get('file'):
         res['path'] = str(ds.pathobj / PurePosixPath(d['file']))
     if 'command' in d:
         res['action'] = d['command']
diff --git a/datalad/interface/shell_completion.py b/datalad/interface/shell_completion.py
new file mode 100644
index 000000000..4c165fe69
--- /dev/null
+++ b/datalad/interface/shell_completion.py
@@ -0,0 +1,78 @@
+# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
+# ex: set sts=4 ts=4 sw=4 noet:
+# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
+#
+#   See COPYING file distributed along with the datalad package for the
+#   copyright and license terms.
+#
+# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
+"""A helper command to enable shell (bash, zsh) completion for DataLad
+
+"""
+__docformat__ = 'restructuredtext'
+
+
+from .base import Interface
+from datalad.interface.base import build_doc
+from datalad.interface.utils import eval_results
+from datalad.interface.results import get_status_dict
+
+
+@build_doc
+class ShellCompletion(Interface):
+    """Display shell script for enabling shell completion for DataLad.
+
+    Output of this command should be "sourced" by the bash or zsh to enable
+    shell completions provided by argcomplete.
+
+    Example:
+
+        $ source <(datalad shell-completion)
+        $ datalad --<PRESS TAB to display available option>
+
+    """
+    # XXX prevent common args from being added to the docstring
+    _no_eval_results = True
+    result_renderer = 'tailored'
+    _params_ = {}
+
+    @staticmethod
+    @eval_results
+    def __call__():
+        """
+        """
+        content = """\
+# Universal completion script for DataLad with the core autogenerated by
+# python-argcomplete and only slightly improved to work for ZSH if sourced under ZSH.
+#
+# Instead of just running this command and seeing this output, do
+#
+#    source <(datalad shell-completion)
+#
+# in your bash or zsh session.
+
+if [ "${ZSH_VERSION:-}" != "" ]; then
+  autoload -U compinit && compinit
+  autoload -U bashcompinit && bashcompinit
+fi
+
+_python_argcomplete() {
+    local IFS=''
+    COMPREPLY=( $(IFS="$IFS" COMP_LINE="$COMP_LINE" COMP_POINT="$COMP_POINT" _ARGCOMPLETE_COMP_WORDBREAKS="$COMP_WORDBREAKS" _ARGCOMPLETE=1                   "$1" 8>&1 9>&2 1>/dev/null 2>/dev/null) )
+    if [[ $? != 0 ]]; then
+        unset COMPREPLY
+    fi
+}
+
+complete -o nospace -o default -F _python_argcomplete "datalad"
+"""
+        yield get_status_dict(
+            action='shell_completion',
+            status='ok',
+            content=content
+        )
+
+    @staticmethod
+    def custom_result_renderer(res, **kwargs):
+        from datalad.ui import ui
+        ui.message(res['content'])
diff --git a/docs/source/cmdline.rst b/docs/source/cmdline.rst
index b428b28b5..cc7c5b89b 100644
--- a/docs/source/cmdline.rst
+++ b/docs/source/cmdline.rst
@@ -67,6 +67,7 @@ Miscellaneous commands
    generated/man/datalad-clean
    generated/man/datalad-download-url
    generated/man/datalad-ls
+   generated/man/datalad-shell-completion
    generated/man/datalad-test
    generated/man/datalad-export-archive-ora
 


Test patch:
diff --git a/datalad/interface/tests/test_results.py b/datalad/interface/tests/test_results.py
new file mode 100644
index 000000000..d573f34e0
--- /dev/null
+++ b/datalad/interface/tests/test_results.py
@@ -0,0 +1,46 @@
+# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
+# ex: set sts=4 ts=4 sw=4 noet:
+# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
+#
+#   See COPYING file distributed along with the datalad package for the
+#   copyright and license terms.
+#
+# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
+"""Result utility tests
+
+"""
+
+from datalad.interface.results import (
+    annexjson2result,
+)
+from datalad.distribution.dataset import Dataset
+
+from datalad.tests.utils import (
+    eq_,
+    with_tempfile,
+)
+
+
+@ with_tempfile
+def test_annexjson2result(dspath):
+    # no explicit success means 'error'
+    eq_(annexjson2result(dict(), None),
+        dict(status='error'))
+    # unrecognized -> error
+    eq_(annexjson2result(dict(success='random'), None),
+        dict(status='error'))
+    # success is possible ;-)
+    eq_(annexjson2result(dict(success=True), None),
+        dict(status='ok'))
+
+    # path handling
+    # needs a dataset
+    ds = Dataset(dspath)
+    eq_(annexjson2result(dict(file='file1'), ds),
+        dict(status='error',
+             path=str(ds.pathobj / 'file1')))
+    # on all platforms, paths are reported in platform conventions
+    # although git-annex reports in posix
+    eq_(annexjson2result(dict(file='dir1/file1'), ds),
+        dict(status='error',
+             path=str(ds.pathobj / 'dir1' / 'file1')))
diff --git a/datalad/interface/tests/test_shell_completion.py b/datalad/interface/tests/test_shell_completion.py
new file mode 100644
index 000000000..4ece7833c
--- /dev/null
+++ b/datalad/interface/tests/test_shell_completion.py
@@ -0,0 +1,45 @@
+# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
+# ex: set sts=4 ts=4 sw=4 noet:
+# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
+#
+#   See COPYING file distributed along with the datalad package for the
+#   copyright and license terms.
+#
+# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
+"""test command datalad shell_completion
+
+"""
+
+__docformat__ = 'restructuredtext'
+
+# Not really worth to be there but it is ATM, so let's use that
+from datalad.api import (
+    shell_completion
+)
+from datalad.cmd import WitlessRunner
+from datalad.tests.utils import (
+    assert_cwd_unchanged,
+    eq_,
+    skip_if_on_windows,
+    swallow_outputs,
+)
+
+
+@assert_cwd_unchanged
+def test_shell_completion_python():
+    # largely a smoke test for our print("hello world")
+    with swallow_outputs() as cmo:
+        res = shell_completion()
+        out = cmo.out.rstrip()
+    # we get it printed and returned for double-pleasure
+    eq_(out, res[0]['content'].rstrip())
+
+
+@skip_if_on_windows  # TODO: make it more specific since might work if bash is available
+def test_shell_completion_source():
+    # just smoke test that produced shell script sources nicely without error
+    WitlessRunner().run(['bash', '-c', 'source <(datalad shell-completion)'])
+    # ideally we should feed that shell with TAB to see the result of completion but
+    # yoh sees no easy way ATM, and googled up
+    # https://stackoverflow.com/questions/9137245/unit-test-for-bash-completion-script
+    # requires too much enthusiasm toward this goal.
\ No newline at end of file
diff --git a/datalad/plugin/tests/test_plugins.py b/datalad/plugin/tests/test_plugins.py
index 6a042b089..11f33fe6c 100644
--- a/datalad/plugin/tests/test_plugins.py
+++ b/datalad/plugin/tests/test_plugins.py
@@ -191,13 +191,30 @@ def test_no_annex(path):
     # add inannex pre configuration
     ds.save(opj('code', 'inannex'))
     no_annex(pattern=['code/**', 'README'], dataset=ds.path)
+
+    inannex = (ds.pathobj / 'code' / 'inannex')
+    # Ensure that notinannex's mtime is as recent or newer than .git/index's so
+    # that, on an adjusted branch, the clean filter runs on the next save. This
+    # avoids a racy failure of the managed-branch assert_repo_status check
+    # below.
+    inannex.touch()
+
     # add inannex and README post configuration
     ds.save([opj('code', 'notinannex'), 'README'])
-    assert_repo_status(ds.path)
-    # one is annex'ed, the other is not, despite no change in add call
-    # importantly, also .gitattribute is not annexed
-    eq_([opj('code', 'inannex')],
-        [str(Path(p)) for p in ds.repo.get_annexed_files()])
+
+    if ds.repo.is_managed_branch():
+        # For unlocked files, if .gitattributes is configured so that a file
+        # should go to git, an annexed file will switch to be tracked by git
+        # the next time the clean filter runs on it.
+        #
+        # https://git-annex.branchable.com/forum/one-off_unlocked_annex_files_that_go_against_large/
+        assert_repo_status(ds.path, modified=[inannex])
+    else:
+        assert_repo_status(ds.path)
+        # one is annex'ed, the other is not, despite no change in add call
+        # importantly, also .gitattribute is not annexed
+        eq_([opj('code', 'inannex')],
+            [str(Path(p)) for p in ds.repo.get_annexed_files()])
 
 
 _ds_template = {