Repository: iterative/dvc

Issue:
Fix `cache_exists` for RemoteHTTP
As noted by @Suor  in https://github.com/iterative/dvc/pull/2375
Http remote will not be working properly in case of `no_traverse=False`, due to fact that `list_cache_paths` is not implemented for `RemoteHTTP`.


Discussion:
We could simply set `no_traverse = False` in `RemoteHTTP.__init__` permanently, 
Throw an exception if no_traverse is set for HTTP
Add test for this
@efiop , does it make sense to raise an exception? what about defaulting to `traverse`?
Just setting `self.no_traverse = False` on the `RemoteHTTP.__init__` should work as intended.
@mroutis But in that case setting that option in config will have no affect on it and user won't even know that it is not working.
@efiop , as @pared  noticed, `list_cache_paths` is not implemented for `RemoteHTTP` (neither `gc`). I'm wondering how `HTTP` was used as a cache :thinking: 
@mroutis it is used through no_traverse and has been that way always. gc is not implemented because HTTP doesn't support write, http has been always used as read-only(i.e. pull/fetch only).
@efiop , but what are you going to pull if you haven't `push`ed anything?
@mroutis You could push through s3 and then pull through http, as we've been doing in dvc for a long time already https://github.com/iterative/dvc/blob/master/.dvc/config#L2
Yep, the same we do with `example-get-started` - `dvc remote add --local s3` to push and `dvc remote add http` to then expose it to public as a read-only data repo.

Resolving patch:
diff --git a/dvc/remote/base.py b/dvc/remote/base.py
--- a/dvc/remote/base.py
+++ b/dvc/remote/base.py
@@ -116,7 +116,7 @@ def __init__(self, repo, config):
         )
 
         self.protected = False
-        self.no_traverse = config.get(Config.SECTION_REMOTE_NO_TRAVERSE)
+        self.no_traverse = config.get(Config.SECTION_REMOTE_NO_TRAVERSE, True)
         self._dir_info = {}
 
     def __repr__(self):
diff --git a/dvc/remote/http.py b/dvc/remote/http.py
--- a/dvc/remote/http.py
+++ b/dvc/remote/http.py
@@ -10,7 +10,7 @@
 
 from dvc.progress import progress
 from dvc.exceptions import DvcException
-from dvc.config import Config
+from dvc.config import Config, ConfigError
 from dvc.remote.base import RemoteBASE
 
 
@@ -42,6 +42,12 @@ def __init__(self, repo, config):
         url = config.get(Config.SECTION_REMOTE_URL)
         self.path_info = self.path_cls(url) if url else None
 
+        if not self.no_traverse:
+            raise ConfigError(
+                "HTTP doesn't support traversing the remote to list existing "
+                "files. Use: `dvc remote modify <name> no_traverse true`"
+            )
+
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
         callback = None
         if not no_progress_bar:


Test patch:
diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py
--- a/tests/func/test_data_cloud.py
+++ b/tests/func/test_data_cloud.py
@@ -457,6 +457,21 @@ def setup_method_fixture(self, request, ssh_server):
         self.ssh_server = ssh_server
         self.method_name = request.function.__name__
 
+    def _setup_cloud(self):
+        self._ensure_should_run()
+
+        repo = self._get_url()
+        keyfile = self._get_keyfile()
+
+        config = copy.deepcopy(TEST_CONFIG)
+        config[TEST_SECTION][Config.SECTION_REMOTE_URL] = repo
+        config[TEST_SECTION][Config.SECTION_REMOTE_KEY_FILE] = keyfile
+        config[TEST_SECTION][Config.SECTION_REMOTE_NO_TRAVERSE] = False
+        self.dvc.config.config = config
+        self.cloud = DataCloud(self.dvc)
+
+        self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class())
+
     def _get_url(self):
         user = self.ssh_server.test_creds["username"]
         return get_ssh_url_mocked(user, self.ssh_server.port)
diff --git a/tests/unit/remote/test_http.py b/tests/unit/remote/test_http.py
new file mode 100644
--- /dev/null
+++ b/tests/unit/remote/test_http.py
@@ -0,0 +1,14 @@
+import pytest
+from dvc.remote.http import RemoteHTTP
+from dvc.config import ConfigError
+
+
+def test_no_traverse_compatibility(dvc_repo):
+    config = {
+        "url": "http://example.com/",
+        "path_info": "file.html",
+        "no_traverse": False,
+    }
+
+    with pytest.raises(ConfigError):
+        RemoteHTTP(dvc_repo, config)