Repository: templateflow/python-client

Issue:
BUG: Updating templateflow does not update the repo
For S3-backed systems, this requires 1) identifying that this is an update; 2) unzip new files in the skeleton.

For Datalad-backed systems, this requires 1) identifying that this is an update; 2) `datalad update --merge -r`.

Discussion:

Patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3ec8178..badb362 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -55,6 +55,13 @@ jobs:
             - "/tmp/data/templateflow"
             - "/tmp/cache"
 
+      - run:
+          name: Run tests (config, parameterized TEMPLATEFLOW_USE_DATALAD)
+          command: |
+            pyenv global 3.5.2
+            virtualenv venv
+            pytest -vsx /tmp/src/templateflow/templateflow/conf/tests/test_conf.py
+
       - run:
           name: Run tests (w/ DataLad)
           environment:
@@ -63,7 +70,7 @@ jobs:
             unset TEMPLATEFLOW_HOME
             pyenv global 3.5.2
             virtualenv venv
-            pytest -vsx --doctest-modules /tmp/src/templateflow/templateflow
+            pytest -vsx --doctest-modules /tmp/src/templateflow/templateflow --ignore=/tmp/src/templateflow/templateflow/conf/test_conf.py
 
       - run:
           name: Update TemplateFlow skeleton
@@ -88,7 +95,7 @@ jobs:
             pyenv global 3.5.2
             virtualenv venv
             export TEMPLATEFLOW_HOME="/tmp/data/templateflow"
-            pytest -vsx --doctest-modules /tmp/src/templateflow/templateflow
+            pytest -vsx --doctest-modules /tmp/src/templateflow/templateflow --ignore=/tmp/src/templateflow/templateflow/conf/test_conf.py
 
       - run:
           name: Run tests (pulling from S3)
@@ -98,7 +105,7 @@ jobs:
           command: |
             pyenv global 3.5.2
             virtualenv venv
-            pytest -vsx --doctest-modules /tmp/src/templateflow/templateflow
+            pytest -vsx --doctest-modules /tmp/src/templateflow/templateflow --ignore=/tmp/src/templateflow/templateflow/conf/test_conf.py
 
       - run:
           name: Run tests (w/ DataLad, bypassed via S3)
@@ -112,7 +119,7 @@ jobs:
             pip install -e .
             python -c "from templateflow import api"
             export TEMPLATEFLOW_USE_DATALAD=0
-            pytest -vsx --doctest-modules /tmp/src/templateflow/templateflow
+            pytest -vsx --doctest-modules /tmp/src/templateflow/templateflow --ignore=/tmp/src/templateflow/templateflow/conf/test_conf.py
 
       - run:
           name: Test packaging
@@ -120,8 +127,6 @@ jobs:
             pyenv global 3.5.2
             virtualenv venv
             cd /tmp/src/templateflow
-            echo "${CIRCLE_TAG}" > templateflow/VERSION
-            echo "include templateflow/VERSION" >> MANIFEST.in
             python setup.py sdist
             twine check dist/*
 
@@ -154,8 +159,6 @@ jobs:
             virtualenv venv
             pip install "setuptools>=27.0" twine
             cp /tmp/resources/templateflow-skel.zip templateflow/conf/
-            echo "${CIRCLE_TAG}" > templateflow/VERSION
-            echo "include templateflow/VERSION" >> MANIFEST.in
             python setup.py sdist
             twine check dist/*
             twine upload dist/*
diff --git a/setup.py b/setup.py
index 404ccf9..de35d3c 100644
--- a/setup.py
+++ b/setup.py
@@ -1,13 +1,37 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 """ templateflow setup script """
+from setuptools.command.install import install
+from setuptools.command.develop import develop
 
 
-def main():
+def make_cmdclass(basecmd):
+    """Decorate setuptools commands."""
+    base_run = basecmd.run
+
+    def new_run(self):
+        from templateflow.conf import update_home
+        update_home()
+        base_run(self)
+
+    basecmd.run = new_run
+    return basecmd
+
+
+@make_cmdclass
+class CheckHomeDevCommand(develop):
+    pass
+
+@make_cmdclass
+class CheckHomeProdCommand(install):
+    pass
+
+
+if __name__ == '__main__':
     """ Install entry-point """
-    from os import path as op
-    from inspect import getfile, currentframe
     from setuptools import setup, find_packages
+    from versioneer import get_cmdclass, get_version
+
     from templateflow.__about__ import (
         __packagename__,
         __author__,
@@ -26,27 +50,9 @@ def main():
         EXTRA_REQUIRES,
     )
 
-    pkg_data = {__packagename__: [
-        'conf/config.json',
-        'conf/templateflow-skel.zip'
-    ]}
-
-    root_dir = op.dirname(op.abspath(getfile(currentframe())))
-    version = None
-    cmdclass = {}
-    if op.isfile(op.join(root_dir, __packagename__, 'VERSION')):
-        with open(op.join(root_dir, __packagename__, 'VERSION')) as vfile:
-            version = vfile.readline().strip()
-        pkg_data[__packagename__].insert(0, 'VERSION')
-
-    if version is None:
-        import versioneer
-        version = versioneer.get_version()
-        cmdclass = versioneer.get_cmdclass()
-
     setup(
         name=__packagename__,
-        version=version,
+        version=get_version(),
         description=__description__,
         long_description=__longdesc__,
         author=__author__,
@@ -66,11 +72,13 @@ def main():
         tests_require=TESTS_REQUIRES,
         extras_require=EXTRA_REQUIRES,
         # Data
-        package_data=pkg_data,
         include_package_data=True,
-        cmdclass=cmdclass,
+        package_data={__packagename__: [
+            'conf/config.json',
+            'conf/templateflow-skel.zip'
+        ]},
+        cmdclass=get_cmdclass(cmdclass={
+            'develop': CheckHomeDevCommand,
+            'install': CheckHomeProdCommand,
+        }),
     )
-
-
-if __name__ == '__main__':
-    main()
diff --git a/templateflow/__init__.py b/templateflow/__init__.py
index 578f694..3bd3d15 100644
--- a/templateflow/__init__.py
+++ b/templateflow/__init__.py
@@ -22,3 +22,7 @@ __all__ = [
     '__description__',
     '__longdesc__',
 ]
+
+from ._version import get_versions
+__version__ = get_versions()['version']
+del get_versions
diff --git a/templateflow/_version.py b/templateflow/_version.py
index 469e863..41475a3 100644
--- a/templateflow/_version.py
+++ b/templateflow/_version.py
@@ -495,19 +495,6 @@ def get_versions():
         # versionfile_source is the relative path from the top of the source
         # tree (where the .git directory might live) to this file. Invert
         # this to find the root from __file__.
-        root_dir = os.path.dirname(root)
-        if os.path.isfile(os.path.join(root_dir, 'VERSION')):
-            with open(os.path.join(root_dir, 'VERSION')) as vfile:
-                version = vfile.readline().strip()
-
-            return {
-                "version": version,
-                "full-revisionid": None,
-                "dirty": None,
-                "error": None,
-                "date": None
-            }
-
         for i in cfg.versionfile_source.split('/'):
             root = os.path.dirname(root)
     except NameError:
diff --git a/templateflow/conf/__init__.py b/templateflow/conf/__init__.py
index 5432e16..24b6df7 100644
--- a/templateflow/conf/__init__.py
+++ b/templateflow/conf/__init__.py
@@ -12,16 +12,16 @@ TF_HOME = Path(getenv('TEMPLATEFLOW_HOME', str(TF_DEFAULT_HOME)))
 TF_GITHUB_SOURCE = 'https://github.com/templateflow/templateflow.git'
 TF_S3_ROOT = 'https://templateflow.s3.amazonaws.com'
 TF_USE_DATALAD = getenv('TEMPLATEFLOW_USE_DATALAD', 'false').lower() in (
-    'true', 'on', '1')
-
-_msg = """\
-TemplateFlow: repository not found at %s. Populating a TemplateFlow stub.
-If the path reported above is not the desired location for TemplateFlow, \
-please set the TEMPLATEFLOW_HOME environment variable.
-""" % TF_HOME
+    'true', 'on', '1', 'yes', 'y')
+TF_CACHED = True
 
 if not TF_HOME.exists() or not list(TF_HOME.iterdir()):
-    warn(_msg, ResourceWarning)
+    TF_CACHED = False
+    warn("""\
+TemplateFlow: repository not found at %s. Populating a new TemplateFlow stub.
+If the path reported above is not the desired location for TemplateFlow, \
+please set the TEMPLATEFLOW_HOME environment variable.\
+""" % TF_HOME, ResourceWarning)
     if TF_USE_DATALAD:
         try:
             from datalad.api import install
@@ -38,6 +38,44 @@ if not TF_HOME.exists() or not list(TF_HOME.iterdir()):
                                        'conf/templateflow-skel.zip'), 'r') as zipref:
             zipref.extractall(str(TF_HOME))
 
+
+def update_home(force=False):
+    """Update an existing DataLad or S3 home."""
+    if not force and not TF_CACHED:
+        print("""\
+TemplateFlow was not cached (TEMPLATEFLOW_HOME=%s), \
+a fresh initialization was done.""" % TF_HOME)
+        return False
+
+    if TF_USE_DATALAD:
+        from datalad.api import update
+        print("Updating TemplateFlow's HOME using DataLad ...")
+        try:
+            update(str(TF_HOME), recursive=True, merge=True)
+        except Exception as e:
+            warn("""Error updating TemplateFlow's home directory (using DataLad):
+%s""" % str(e))
+        return True
+
+    # This is an S3 type of installation
+    from zipfile import ZipFile
+    with ZipFile(resource_filename('templateflow',
+                                   'conf/templateflow-skel.zip'), 'r') as zipref:
+        allfiles = zipref.namelist()
+        current_files = [s.relative_to(TF_HOME) for s in TF_HOME.glob('**/*')]
+        existing = sorted(set(['%s/' % s.parent for s in current_files])) + \
+            [str(s) for s in current_files]
+        newfiles = sorted(set(allfiles) - set(existing))
+        if newfiles:
+            print("Updating TemplateFlow's HOME using S3. "
+                  "Adding: \n%s" % "\n".join(newfiles))
+            zipref.extractall(str(TF_HOME), members=newfiles)
+            return True
+
+    print("TemplateFlow's HOME directory (S3 type) was up-to-date.")
+    return False
+
+
 TF_LAYOUT = Layout(
     str(TF_HOME), validate=False, config='templateflow',
     ignore=['.git', '.datalad', '.gitannex', '.gitattributes', 'scripts'])
diff --git a/versioneer.py b/versioneer.py
index 5e26e65..d5471f9 100644
--- a/versioneer.py
+++ b/versioneer.py
@@ -339,7 +339,7 @@ def get_config_from_root(root):
     # configparser.NoOptionError (if it lacks "VCS="). See the docstring at
     # the top of versioneer.py for instructions on writing your setup.cfg .
     setup_cfg = os.path.join(root, "setup.cfg")
-    parser = configparser.ConfigParser()
+    parser = configparser.SafeConfigParser()
     with open(setup_cfg, "r") as f:
         parser.readfp(f)
     VCS = parser.get("versioneer", "VCS")  # mandatory
@@ -1480,8 +1480,11 @@ def get_version():
     return get_versions()["version"]
 
 
-def get_cmdclass():
-    """Get the custom setuptools/distutils subclasses used by Versioneer."""
+def get_cmdclass(cmdclass=None):
+    """Get the custom setuptools/distutils subclasses used by Versioneer.
+    If the package uses a different cmdclass (e.g. one from numpy), it
+    should be provide as an argument.
+    """
     if "versioneer" in sys.modules:
         del sys.modules["versioneer"]
         # this fixes the "python setup.py develop" case (also 'install' and
@@ -1497,7 +1500,7 @@ def get_cmdclass():
         # happens, we protect the child from the parent's versioneer too.
         # Also see https://github.com/warner/python-versioneer/issues/52
 
-    cmds = {}
+    cmds = {} if cmdclass is None else cmdclass.copy()
 
     # we add "version" to both distutils and setuptools
     from distutils.core import Command
@@ -1539,7 +1542,9 @@ def get_cmdclass():
     #  setup.py egg_info -> ?
 
     # we override different "build_py" commands for both environments
-    if "setuptools" in sys.modules:
+    if 'build_py' in cmds:
+        _build_py = cmds['build_py']
+    elif "setuptools" in sys.modules:
         from setuptools.command.build_py import build_py as _build_py
     else:
         from distutils.command.build_py import build_py as _build_py
@@ -1620,7 +1625,9 @@ def get_cmdclass():
         cmds["py2exe"] = cmd_py2exe
 
     # we override different "sdist" commands for both environments
-    if "setuptools" in sys.modules:
+    if 'sdist' in cmds:
+        _sdist = cmds['sdist']
+    elif "setuptools" in sys.modules:
         from setuptools.command.sdist import sdist as _sdist
     else:
         from distutils.command.sdist import sdist as _sdist


Test patch:
diff --git a/templateflow/conf/tests/__init__.py b/templateflow/conf/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/templateflow/conf/tests/test_conf.py b/templateflow/conf/tests/test_conf.py
new file mode 100644
index 0000000..274d14d
--- /dev/null
+++ b/templateflow/conf/tests/test_conf.py
@@ -0,0 +1,83 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
+# vi: set ft=python sts=4 ts=4 sw=4 et:
+import pytest
+from shutil import rmtree
+from importlib import reload
+from templateflow import conf as tfc
+
+
+@pytest.mark.parametrize('use_datalad', ['off', 'on'])
+def test_conf_init(monkeypatch, tmp_path, capsys, use_datalad):
+    """Check the correct functioning of config set-up."""
+    home = (tmp_path / '-'.join(('tf', 'dl', use_datalad))).resolve()
+    monkeypatch.setenv('TEMPLATEFLOW_USE_DATALAD', use_datalad)
+    monkeypatch.setenv('TEMPLATEFLOW_HOME', str(home))
+
+    # First execution, the S3 stub is created (or datalad install)
+    reload(tfc)
+    assert tfc.TF_CACHED is False
+    assert str(tfc.TF_HOME) == str(home)
+
+    reload(tfc)
+    assert tfc.TF_CACHED is True
+    assert str(tfc.TF_HOME) == str(home)
+
+
+@pytest.mark.parametrize('use_datalad', ['off', 'on'])
+def test_update_home(monkeypatch, tmp_path, capsys, use_datalad):
+    """Check the correct functioning of the installation hook."""
+    home = (tmp_path / '-'.join(('tf', 'dl', use_datalad))).resolve()
+    monkeypatch.setenv('TEMPLATEFLOW_USE_DATALAD', use_datalad)
+    monkeypatch.setenv('TEMPLATEFLOW_HOME', str(home))
+
+    reload(tfc)
+    # First execution, the S3 stub is created (or datalad install)
+    assert tfc.TF_CACHED is False
+    assert tfc.update_home() is False
+    out = capsys.readouterr()[0]
+    assert out.startswith('TemplateFlow was not cached')
+    assert ('TEMPLATEFLOW_HOME=%s' % home) in out
+    assert home.exists()
+    assert len(list(home.iterdir())) > 0
+
+    updated = tfc.update_home(force=True)  # Templateflow is now cached
+    out = capsys.readouterr()[0]
+    assert not out.startswith('TemplateFlow was not cached')
+
+    if use_datalad == 'on':
+        assert out.startswith("Updating TemplateFlow's HOME using DataLad")
+        assert updated is True
+
+    elif use_datalad == 'off':
+        # At this point, S3 should be up-to-date
+        assert updated is False
+        assert out.startswith("TemplateFlow's HOME directory (S3 type) was up-to-date.")
+
+        # Let's force an update
+        rmtree(str(home / 'tpl-MNI152NLin2009cAsym'))
+        updated = tfc.update_home(force=True)
+        out = capsys.readouterr()[0]
+        assert updated is True
+        assert out.startswith("Updating TemplateFlow's HOME using S3.")
+
+    reload(tfc)
+    assert tfc.TF_CACHED is True
+    updated = tfc.update_home()  # Templateflow is now cached
+    out = capsys.readouterr()[0]
+    assert not out.startswith('TemplateFlow was not cached')
+
+    if use_datalad == 'on':
+        assert out.startswith("Updating TemplateFlow's HOME using DataLad")
+        assert updated is True
+
+    elif use_datalad == 'off':
+        # At this point, S3 should be up-to-date
+        assert updated is False
+        assert out.startswith("TemplateFlow's HOME directory (S3 type) was up-to-date.")
+
+        # Let's force an update
+        rmtree(str(home / 'tpl-MNI152NLin2009cAsym'))
+        updated = tfc.update_home()
+        out = capsys.readouterr()[0]
+        assert updated is True
+        assert out.startswith("Updating TemplateFlow's HOME using S3.")