Repository: datalad/datalad

Issue:
test_expanduser fails on master/windows on github CI
Just spotted within datalad/git-annex sweep of tests (ok on released, maint):
Tests were passing 5 days back, so possibly triggered by #5310

https://github.com/datalad/git-annex/runs/1722165033?check_suite_focus=true

```
======================================================================
ERROR: datalad.core.distributed.tests.test_clone.test_expanduser
----------------------------------------------------------------------
Traceback (most recent call last):
  File "C:\hostedtoolcache\windows\Python\3.7.9\x64\lib\site-packages\nose\case.py", line 198, in runTest
    self.test(*self.arg)
  File "C:\hostedtoolcache\windows\Python\3.7.9\x64\lib\site-packages\datalad\tests\utils.py", line 757, in _wrap_with_tempfile
    return t(*(arg + (filename,)), **kw)
  File "C:\hostedtoolcache\windows\Python\3.7.9\x64\lib\site-packages\datalad\tests\utils.py", line 757, in _wrap_with_tempfile
    return t(*(arg + (filename,)), **kw)
  File "C:\hostedtoolcache\windows\Python\3.7.9\x64\lib\site-packages\datalad\core\distributed\tests\test_clone.py", line 545, in test_expanduser
    cloneds = clone(op.join('~', 'src'), 'dest')
  File "C:\hostedtoolcache\windows\Python\3.7.9\x64\lib\site-packages\datalad\interface\utils.py", line 482, in eval_func
    return return_func(generator_func)(*args, **kwargs)
  File "C:\hostedtoolcache\windows\Python\3.7.9\x64\lib\site-packages\datalad\interface\utils.py", line 470, in return_func
    results = list(results)
  File "C:\hostedtoolcache\windows\Python\3.7.9\x64\lib\site-packages\datalad\interface\utils.py", line 457, in generator_func
    msg="Command did not complete successfully")
datalad.support.exceptions.IncompleteResultsError: Command did not complete successfully. 1 failed:

[{'action': 'install',
  'message': ('Failed to clone from any candidate source URL. Encountered '
              'errors per each url were:\n'
              '- %s',
              '~\\src\n'
              '  CommandError: \'"git" "clone" "--progress" '
              '"C:\\Users\\runneradmin\\AppData\\Local\\Temp\\datalad_temp_zl9rmwun\\src" '
              '"C:\\Users\\runneradmin\\AppData\\Local\\Temp\\datalad_temp_4aq19uux\\dest"\' '
              "failed with exitcode 128 [err: 'Cloning into "
              "'C:\\Users\\runneradmin\\AppData\\Local\\Temp\\datalad_temp_4aq19uux\\dest'...\n"
              'fatal: '
              "'C:\\Users\\runneradmin\\AppData\\Local\\Temp\\datalad_temp_zl9rmwun\\src' "
              'does not appear to be a git repository\n'
              'fatal: Could not read from remote repository.\n'
              '\n'
              'Please make sure you have the correct access rights\n'
              "and the repository exists.']"),
  'path': 'C:\\Users\\runneradmin\\AppData\\Local\\Temp\\datalad_temp_4aq19uux\\dest',
  'source_url': '~\\src',
  'status': 'error',
  'type': 'dataset'}]
```

Discussion:

Patch:
diff --git a/datalad/__init__.py b/datalad/__init__.py
index 7fe201c69..0fca35897 100644
--- a/datalad/__init__.py
+++ b/datalad/__init__.py
@@ -47,7 +47,12 @@ from .config import ConfigManager
 cfg = ConfigManager()
 
 from .log import lgr
-from datalad.utils import get_encoding_info, get_envvars_info, getpwd
+from datalad.utils import (
+    get_encoding_info,
+    get_envvars_info,
+    get_home_envvars,
+    getpwd,
+)
 
 # To analyze/initiate our decision making on what current directory to return
 getpwd()
@@ -88,36 +93,41 @@ test.__test__ = False
 # To store settings which setup_package changes and teardown_package should return
 _test_states = {
     'loglevel': None,
-    'DATALAD_LOG_LEVEL': None,
-    'HOME': None,
+    'env': {},
 }
 
 
 def setup_package():
     import os
     from datalad import consts
-    _test_states['HOME'] = os.environ.get('HOME', None)
-    _test_states['DATASETS_TOPURL_ENV'] = os.environ.get('DATALAD_DATASETS_TOPURL', None)
+
+    _test_states['env'] = {}
+
+    def set_envvar(v, val):
+        """Memoize and then set env var"""
+        _test_states['env'][v] = os.environ.get(v, None)
+        os.environ[v] = val
+
     _test_states['DATASETS_TOPURL'] = consts.DATASETS_TOPURL
-    os.environ['DATALAD_DATASETS_TOPURL'] = consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/'
+    consts.DATASETS_TOPURL = 'http://datasets-tests.datalad.org/'
+    set_envvar('DATALAD_DATASETS_TOPURL', consts.DATASETS_TOPURL)
 
     from datalad.tests.utils import DEFAULT_BRANCH
-    _test_states["GIT_CONFIG_PARAMETERS"] = os.environ.get(
-        "GIT_CONFIG_PARAMETERS")
-    os.environ["GIT_CONFIG_PARAMETERS"] = "'init.defaultBranch={}'".format(
-        DEFAULT_BRANCH)
+    set_envvar("GIT_CONFIG_PARAMETERS", "'init.defaultBranch={}'".format(DEFAULT_BRANCH))
 
     # To overcome pybuild overriding HOME but us possibly wanting our
     # own HOME where we pre-setup git for testing (name, email)
     if 'GIT_HOME' in os.environ:
-        os.environ['HOME'] = os.environ['GIT_HOME']
+        set_envvar('HOME', os.environ['GIT_HOME'])
     else:
         # we setup our own new HOME, the BEST and HUGE one
         from datalad.utils import make_tempfile
         from datalad.tests import _TEMP_PATHS_GENERATED
         # TODO: split into a function + context manager
         with make_tempfile(mkdir=True) as new_home:
-            os.environ['HOME'] = new_home
+            pass
+        for v, val in get_home_envvars(new_home).items():
+            set_envvar(v, val)
         if not os.path.exists(new_home):
             os.makedirs(new_home)
         with open(os.path.join(new_home, '.gitconfig'), 'w') as f:
@@ -155,8 +165,7 @@ def setup_package():
         lgr.setLevel(100)
 
         # And we should also set it within environ so underlying commands also stay silent
-        _test_states['DATALAD_LOG_LEVEL'] = DATALAD_LOG_LEVEL
-        os.environ['DATALAD_LOG_LEVEL'] = '100'
+        set_envvar('DATALAD_LOG_LEVEL', '100')
     else:
         # We are not overriding them, since explicitly were asked to have some log level
         _test_states['loglevel'] = None
@@ -213,10 +222,6 @@ def teardown_package():
     ui.set_backend(_test_states['ui_backend'])
     if _test_states['loglevel'] is not None:
         lgr.setLevel(_test_states['loglevel'])
-        if _test_states['DATALAD_LOG_LEVEL'] is None:
-            os.environ.pop('DATALAD_LOG_LEVEL')
-        else:
-            os.environ['DATALAD_LOG_LEVEL'] = _test_states['DATALAD_LOG_LEVEL']
 
     from datalad.tests import _TEMP_PATHS_GENERATED
     if len(_TEMP_PATHS_GENERATED):
@@ -227,14 +232,12 @@ def teardown_package():
     for path in _TEMP_PATHS_GENERATED:
         rmtemp(path, ignore_errors=True)
 
-    if _test_states['HOME'] is not None:
-        os.environ['HOME'] = _test_states['HOME']
-
-    git_config_params = _test_states["GIT_CONFIG_PARAMETERS"]
-    if git_config_params is None:
-        os.environ.pop("GIT_CONFIG_PARAMETERS")
-    else:
-        os.environ["GIT_CONFIG_PARAMETERS"] = git_config_params
+    # restore all the env variables
+    for v, val in _test_states['env'].items():
+        if val is not None:
+            os.environ[v] = val
+        else:
+            os.environ.pop(v)
 
     # Re-establish correct global config after changing $HOME.
     # Might be superfluous, since after teardown datalad.cfg shouldn't be
@@ -242,8 +245,6 @@ def teardown_package():
     # either way.
     cfg.reload(force=True)
 
-    if _test_states['DATASETS_TOPURL_ENV']:
-        os.environ['DATALAD_DATASETS_TOPURL'] = _test_states['DATASETS_TOPURL_ENV']
     consts.DATASETS_TOPURL = _test_states['DATASETS_TOPURL']
 
     from datalad.support.cookies import cookies_db
diff --git a/datalad/utils.py b/datalad/utils.py
index cc99c4135..c1a340f57 100644
--- a/datalad/utils.py
+++ b/datalad/utils.py
@@ -183,6 +183,27 @@ def not_supported_on_windows(msg=None):
                                   + (": %s" % msg if msg else ""))
 
 
+def get_home_envvars(new_home):
+    """Return dict with env variables to be adjusted for a new HOME
+
+    Only variables found in current os.environ are adjusted.
+
+    Parameters
+    ----------
+    new_home: str
+      New home path, in native to OS "schema"
+    """
+    environ = os.environ
+    out = {'HOME': new_home}
+    if on_windows:
+        # requires special handling, since it has a number of relevant variables
+        # and also Python changed its behavior and started to respect USERPROFILE only
+        # since python 3.8: https://bugs.python.org/issue36264
+        out['USERPROFILE'] = new_home
+        out['HOMEDRIVE'], out['HOMEPATH'] = op.splitdrive(new_home)
+    return {v: val for v, val in out.items() if v in os.environ}
+
+
 def shortened_repr(value, l=30):
     try:
         if hasattr(value, '__repr__') and (value.__repr__ is not object.__repr__):


Test patch:
diff --git a/datalad/core/distributed/tests/test_clone.py b/datalad/core/distributed/tests/test_clone.py
index 084b6b74a..9e92abef3 100644
--- a/datalad/core/distributed/tests/test_clone.py
+++ b/datalad/core/distributed/tests/test_clone.py
@@ -24,6 +24,7 @@ from datalad.api import (
 )
 from datalad.utils import (
     chpwd,
+    get_home_envvars,
     Path,
     on_windows,
     rmtree
@@ -525,7 +526,7 @@ def test_expanduser(srcpath, destpath):
     src = Dataset(Path(srcpath) / 'src').create()
     dest = Dataset(Path(destpath) / 'dest').create()
 
-    with chpwd(destpath), patch.dict('os.environ', {'HOME': srcpath}):
+    with chpwd(destpath), patch.dict('os.environ', get_home_envvars(srcpath)):
         res = clone(op.join('~', 'src'), 'dest', result_xfm=None, return_type='list',
                     on_failure='ignore')
         assert_result_count(res, 1)
diff --git a/datalad/tests/test_base.py b/datalad/tests/test_base.py
index 8255bdfeb..f837e2a84 100644
--- a/datalad/tests/test_base.py
+++ b/datalad/tests/test_base.py
@@ -10,6 +10,8 @@
 import os
 import os.path as op
 
+from datalad.utils import get_home_envvars
+
 from datalad.tests.utils import (
     assert_in,
     assert_raises,
@@ -72,7 +74,7 @@ def test_git_config_warning(path):
     if 'GIT_AUTHOR_NAME' in os.environ:
         raise SkipTest("Found existing explicit identity config")
     with chpwd(path), \
-            patch.dict('os.environ', {'HOME': path}), \
+            patch.dict('os.environ', get_home_envvars(path)), \
             swallow_logs(new_level=30) as cml:
         # no configs in that empty HOME
         from datalad.api import Dataset
diff --git a/datalad/tests/test_config.py b/datalad/tests/test_config.py
index 103cced0e..f77b4b504 100644
--- a/datalad/tests/test_config.py
+++ b/datalad/tests/test_config.py
@@ -31,6 +31,7 @@ from datalad.tests.utils import (
     with_tree,
 )
 from datalad.utils import (
+    get_home_envvars,
     swallow_logs,
     Path
 )
@@ -177,8 +178,9 @@ def test_something(path, new_home):
     # very carefully test non-local config
     # so carefully that even in case of bad weather Yarik doesn't find some
     # lame datalad unittest sections in his precious ~/.gitconfig
+    env = get_home_envvars(new_home)
     with patch.dict('os.environ',
-                    {'HOME': new_home, 'DATALAD_SNEAKY_ADDITION': 'ignore'}):
+                    dict(get_home_envvars(new_home), DATALAD_SNEAKY_ADDITION='ignore')):
         global_gitconfig = opj(new_home, '.gitconfig')
         assert(not exists(global_gitconfig))
         globalcfg = ConfigManager()
@@ -509,7 +511,7 @@ def test_global_config():
 
     # from within tests, global config should be read from faked $HOME (see
     # setup_package)
-    glb_cfg_file = Path(os.environ['HOME']) / '.gitconfig'
+    glb_cfg_file = Path(os.path.expanduser('~')) / '.gitconfig'
     assert any(glb_cfg_file.samefile(Path(p)) for p in dl_cfg._cfgfiles)
     assert_equal(dl_cfg.get("user.name"), "DataLad Tester")
     assert_equal(dl_cfg.get("user.email"), "test@example.com")