Repository: dandi/dandi-cli

Issue:
use redirector's `/server-info` (or just "OPTIONS" call)
A satellite issue to redirector's https://github.com/dandi/redirector/issues/20 to be merged in after redirector gets its portion merged:

- provide a helper (probably with caching) e.g. `get_instance` to avoid direct use of `known_instances` lookup table.  Instead it should given an instance name (or even just a URL later on?), take a `.redirector` url and request that information from it.  If it is a URL provided -- should just talk to it directly (that would in the future allow to specify "instance" via URL not hardcoded name)
- if there is a response from the redirector providing the record:
  - if current client version does not satisfy `cli-version` in the record, raise a dedicated exception which would explain that the instance demands minimal version X and there are some bad versions which must not be used (if any prescribed in that record). 
  - compose instance record using `.url` fields from the record (instead of hardcoded ones in `consts.py`)
- if there is no response from the redirector, issue a warning and continue using hard-coded values for now.  later we will RF it away but probably worth keeping it as a safety blanket for now.


Comments:

Patch:
diff --git a/dandi/download.py b/dandi/download.py
index b3cd295a..40b7ce1f 100644
--- a/dandi/download.py
+++ b/dandi/download.py
@@ -6,7 +6,7 @@ from . import girder, get_logger
 from .consts import dandiset_metadata_file, known_instances, metadata_digests
 from .dandiset import Dandiset
 from .exceptions import FailedToConnectError, NotFoundError, UnknownURLError
-from .utils import flatten, flattened, Parallel, delayed
+from .utils import flatten, flattened, Parallel, delayed, get_instance
 
 lgr = get_logger()
 
@@ -121,9 +121,7 @@ class _dandi_url_parser:
                                 settings["map_instance"], ", ".join(known_instances)
                             )
                         )
-                    return (known_instances[settings["map_instance"]].girder,) + tuple(
-                        _
-                    )
+                    return (get_instance(settings["map_instance"]).girder,) + tuple(_)
                 continue  # in this run we ignore an match further
             else:
                 break
diff --git a/dandi/exceptions.py b/dandi/exceptions.py
index ce64b587..3195d53e 100644
--- a/dandi/exceptions.py
+++ b/dandi/exceptions.py
@@ -23,3 +23,34 @@ class FailedToConnectError(RuntimeError):
     """Failed to connect to online resource"""
 
     pass
+
+
+class CliVersionError(RuntimeError):
+    """ Base class for `CliVersionTooOldError` and `BadCliVersionError` """
+
+    def __init__(self, our_version, minversion, bad_versions):
+        self.our_version = our_version
+        self.minversion = minversion
+        self.bad_versions = bad_versions
+
+    def server_requirements(self):
+        s = f"Server requires at least version {self.minversion}"
+        if self.bad_versions:
+            s += f" (but not {', '.join(map(str, self.bad_versions))})"
+        return s
+
+
+class CliVersionTooOldError(CliVersionError):
+    def __str__(self):
+        return (
+            f"Client version {self.our_version} is too old!  "
+            + self.server_requirements()
+        )
+
+
+class BadCliVersionError(CliVersionError):
+    def __str__(self):
+        return (
+            f"Client version {self.our_version} is rejected by server!  "
+            + self.server_requirements()
+        )
diff --git a/dandi/register.py b/dandi/register.py
index 87030489..6b4c3d9b 100644
--- a/dandi/register.py
+++ b/dandi/register.py
@@ -3,14 +3,15 @@ import os.path as op
 
 from . import get_logger
 from . import girder
-from .consts import dandiset_metadata_file, known_instances, routes
+from .consts import dandiset_metadata_file, routes
 from .dandiset import Dandiset
+from .utils import get_instance
 
 lgr = get_logger()
 
 
 def register(name, description, dandiset_path=None, dandi_instance="dandi"):
-    dandi_instance = known_instances[dandi_instance]
+    dandi_instance = get_instance(dandi_instance)
     if not dandiset_path and op.exists(dandiset_metadata_file):
         dandiset = Dandiset.find(os.getcwd())
         if dandiset:
diff --git a/dandi/upload.py b/dandi/upload.py
index edac0c7c..ce40ec85 100644
--- a/dandi/upload.py
+++ b/dandi/upload.py
@@ -8,12 +8,11 @@ import time
 
 from .cli.command import lgr
 from . import __version__
-from .utils import ensure_datetime, ensure_strtime
+from .utils import ensure_datetime, ensure_strtime, get_instance
 from .consts import (
     collection_drafts,
     dandiset_identifier_regex,
     dandiset_metadata_file,
-    known_instances,
     metadata_digests,
 )
 
@@ -91,7 +90,7 @@ def upload(
 
     ignore_benign_pynwb_warnings()  # so validate doesn't whine
 
-    client = girder.get_client(known_instances[dandi_instance].girder)
+    client = girder.get_client(get_instance(dandi_instance).girder)
 
     try:
         collection_rec = girder.ensure_collection(client, girder_collection)
diff --git a/dandi/utils.py b/dandi/utils.py
index 67bb90b5..b6b26e55 100644
--- a/dandi/utils.py
+++ b/dandi/utils.py
@@ -11,7 +11,13 @@ import sys
 
 from pathlib import Path
 
+import requests
 import ruamel.yaml
+from semantic_version import Version
+
+from . import __version__
+from .consts import dandi_instance, known_instances, known_instances_rev
+from .exceptions import BadCliVersionError, CliVersionTooOldError
 
 if sys.version_info[:2] < (3, 7):
     import dateutil.parser
@@ -504,3 +510,50 @@ def delayed(*args, **kwargs):
     import joblib
 
     return joblib.delayed(*args, **kwargs)
+
+
+def get_instance(dandi_instance_id):
+    if dandi_instance_id.lower().startswith(("http://", "https://")):
+        redirector_url = dandi_instance_id
+        dandi_id = known_instances_rev.get(redirector_url)
+        if dandi_id is not None:
+            instance = known_instances[dandi_id]
+        else:
+            instance = None
+    else:
+        instance = known_instances[dandi_instance_id]
+        redirector_url = instance.redirector
+        if redirector_url is None:
+            return instance
+    try:
+        r = requests.get(redirector_url.rstrip("/") + "/server-info")
+        r.raise_for_status()
+    except Exception as e:
+        lgr.warning("Request to %s failed (%s)", redirector_url, str(e))
+        if instance is not None:
+            lgr.warning("Using hard-coded URLs")
+            return instance
+        else:
+            raise RuntimeError(
+                f"Could not retrieve server info from {redirector_url},"
+                " and client does not recognize URL"
+            )
+    server_info = r.json()
+    try:
+        minversion = Version(server_info["cli-minimal-version"])
+        bad_versions = [Version(v) for v in server_info["cli-bad-versions"]]
+    except ValueError as e:
+        raise ValueError(
+            f"{redirector_url} returned an incorrectly formatted version;"
+            f" please contact that server's administrators: {e}"
+        )
+    our_version = Version(__version__)
+    if our_version < minversion:
+        raise CliVersionTooOldError(our_version, minversion, bad_versions)
+    if our_version in bad_versions:
+        raise BadCliVersionError(our_version, minversion, bad_versions)
+    return dandi_instance(
+        girder=server_info["services"]["girder"]["url"],
+        gui=server_info["services"]["webui"]["url"],
+        redirector=redirector_url,
+    )
diff --git a/setup.cfg b/setup.cfg
index a90eb73d..9bf80cbd 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -42,6 +42,7 @@ install_requires =
     joblib
     pyout
     humanize
+    requests ~= 2.20
     ruamel.yaml >=0.15, <1
     keyring
     keyrings.alt
@@ -70,7 +71,7 @@ test =
     mock
     pytest
     pytest-cov
-    requests ~= 2.20
+    responses
 all =
     #%(doc)s
     %(extensions)s
diff --git a/tox.ini b/tox.ini
index be8222d4..1bf60117 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,6 +23,7 @@ filterwarnings =
     error
     ignore:No cached namespaces found .*:UserWarning
     ignore:ignoring namespace '.*' because it already exists:UserWarning
+    ignore::DeprecationWarning:responses
 
 [coverage:run]
 parallel = True


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 4a8ce44d..506d3d2a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -25,8 +25,10 @@ jobs:
     steps:
     - name: Set up environment
       uses: actions/checkout@v1
-      with:  # no need for the history
-        fetch-depth: 1
+      with:
+        # Fetch all commits so that versioneer will return something compatible
+        # with semantic-version
+        fetch-depth: 0
     - name: Set up Python ${{ matrix.python }}
       uses: actions/setup-python@v1
       with:
diff --git a/dandi/tests/data/dandiarchive-docker/docker-compose.yml b/dandi/tests/data/dandiarchive-docker/docker-compose.yml
index ef0a109c..5b825e21 100644
--- a/dandi/tests/data/dandiarchive-docker/docker-compose.yml
+++ b/dandi/tests/data/dandiarchive-docker/docker-compose.yml
@@ -42,7 +42,7 @@ services:
     ports:
       - "8079:8080"
     environment:
-      GIRDER_URL: http://girder:8080
+      GIRDER_URL: http://localhost:8081
       GUI_URL: http://localhost:8086
       ABOUT_URL: http://www.dandiarchive.org
 
diff --git a/dandi/tests/test_utils.py b/dandi/tests/test_utils.py
index 1d7a7736..0048c9a8 100644
--- a/dandi/tests/test_utils.py
+++ b/dandi/tests/test_utils.py
@@ -2,12 +2,17 @@ import inspect
 import os.path as op
 import time
 import pytest
+import responses
+from .. import __version__
+from ..consts import dandi_instance, known_instances
+from ..exceptions import BadCliVersionError, CliVersionTooOldError
 from ..utils import (
     ensure_datetime,
     ensure_strtime,
     find_files,
     flatten,
     flattened,
+    get_instance,
     get_utcnow_datetime,
     is_same_time,
     on_windows,
@@ -121,3 +126,169 @@ def test_flatten():
         0,
         1,
     ]
+
+
+@responses.activate
+def test_get_instance_dandi():
+    responses.add(
+        responses.GET,
+        "https://dandiarchive.org/server-info",
+        json={
+            "version": "1.0.0",
+            "cli-minimal-version": "0.5.0",
+            "cli-bad-versions": [],
+            "services": {
+                "girder": {"url": "https://girder.dandi"},
+                "webui": {"url": "https://gui.dandi"},
+                "api": {"url": "https://publish.dandi/api"},
+                "jupyterhub": {"url": "https://hub.dandi"},
+            },
+        },
+    )
+    assert get_instance("dandi") == dandi_instance(
+        girder="https://girder.dandi",
+        gui="https://gui.dandi",
+        redirector="https://dandiarchive.org",
+    )
+
+
+@responses.activate
+def test_get_instance_url():
+    responses.add(
+        responses.GET,
+        "https://example.dandi/server-info",
+        json={
+            "version": "1.0.0",
+            "cli-minimal-version": "0.5.0",
+            "cli-bad-versions": [],
+            "services": {
+                "girder": {"url": "https://girder.dandi"},
+                "webui": {"url": "https://gui.dandi"},
+                "api": {"url": "https://publish.dandi/api"},
+                "jupyterhub": {"url": "https://hub.dandi"},
+            },
+        },
+    )
+    assert get_instance("https://example.dandi/") == dandi_instance(
+        girder="https://girder.dandi",
+        gui="https://gui.dandi",
+        redirector="https://example.dandi/",
+    )
+
+
+@responses.activate
+def test_get_instance_cli_version_too_old():
+    responses.add(
+        responses.GET,
+        "https://example.dandi/server-info",
+        json={
+            "version": "1.0.0",
+            "cli-minimal-version": "99.99.99",
+            "cli-bad-versions": [],
+            "services": {
+                "girder": {"url": "https://girder.dandi"},
+                "webui": {"url": "https://gui.dandi"},
+                "api": {"url": "https://publish.dandi/api"},
+                "jupyterhub": {"url": "https://hub.dandi"},
+            },
+        },
+    )
+    with pytest.raises(CliVersionTooOldError) as excinfo:
+        get_instance("https://example.dandi/")
+    assert str(excinfo.value) == (
+        f"Client version {__version__} is too old!"
+        "  Server requires at least version 99.99.99"
+    )
+
+
+@responses.activate
+def test_get_instance_bad_cli_version():
+    responses.add(
+        responses.GET,
+        "https://example.dandi/server-info",
+        json={
+            "version": "1.0.0",
+            "cli-minimal-version": "0.5.0",
+            "cli-bad-versions": [__version__],
+            "services": {
+                "girder": {"url": "https://girder.dandi"},
+                "webui": {"url": "https://gui.dandi"},
+                "api": {"url": "https://publish.dandi/api"},
+                "jupyterhub": {"url": "https://hub.dandi"},
+            },
+        },
+    )
+    with pytest.raises(BadCliVersionError) as excinfo:
+        get_instance("https://example.dandi/")
+    assert str(excinfo.value) == (
+        f"Client version {__version__} is rejected by server!"
+        f"  Server requires at least version 0.5.0 (but not {__version__})"
+    )
+
+
+@responses.activate
+def test_get_instance_id_bad_response():
+    responses.add(
+        responses.GET,
+        "https://dandiarchive.org/server-info",
+        body="404 -- not found",
+        status=404,
+    )
+    assert get_instance("dandi") is known_instances["dandi"]
+
+
+@responses.activate
+def test_get_instance_known_url_bad_response():
+    responses.add(
+        responses.GET,
+        "https://dandiarchive.org/server-info",
+        body="404 -- not found",
+        status=404,
+    )
+    assert get_instance("https://dandiarchive.org") is known_instances["dandi"]
+
+
+@responses.activate
+def test_get_instance_unknown_url_bad_response():
+    responses.add(
+        responses.GET,
+        "https://dandi.nil/server-info",
+        body="404 -- not found",
+        status=404,
+    )
+    with pytest.raises(RuntimeError) as excinfo:
+        get_instance("https://dandi.nil")
+    assert str(excinfo.value) == (
+        "Could not retrieve server info from https://dandi.nil,"
+        " and client does not recognize URL"
+    )
+
+
+def test_get_instance_id_no_redirector():
+    assert get_instance("local-girder-only") is known_instances["local-girder-only"]
+
+
+@responses.activate
+def test_get_instance_bad_version_from_server():
+    responses.add(
+        responses.GET,
+        "https://example.dandi/server-info",
+        json={
+            "version": "1.0.0",
+            "cli-minimal-version": "foobar",
+            "cli-bad-versions": [],
+            "services": {
+                "girder": {"url": "https://girder.dandi"},
+                "webui": {"url": "https://gui.dandi"},
+                "api": {"url": "https://publish.dandi/api"},
+                "jupyterhub": {"url": "https://hub.dandi"},
+            },
+        },
+    )
+    with pytest.raises(ValueError) as excinfo:
+        get_instance("https://example.dandi/")
+    assert str(excinfo.value).startswith(
+        "https://example.dandi/ returned an incorrectly formatted version;"
+        " please contact that server's administrators: "
+    )
+    assert "foobar" in str(excinfo.value)