Repository: iterative/dvc

Issue:
 `get-url` and `import-url` doesn't seem to work with S3 buckets anymore.
## Bug Report

1. Create an empty directory
2. `dvc init --no-scm`
3. `dvc import-url s3://some_bucket/some_target -v`
```
2020-07-01 17:14:02,947 DEBUG: fetched: [(3,)]                                  
2020-07-01 17:14:03,123 DEBUG: Removing output 'some_target' of stage: 'some_target.dvc'.
Importing 's3://some_bucket/some_target' -> 'some_target'
2020-07-01 17:14:03,123 DEBUG: Computed stage: 'some_target.dvc' md5: '2f8b87d3b22efd1638f414c3b3f65614'
2020-07-01 17:14:03,123 DEBUG: 'md5' of stage: 'some_target.dvc' changed.
2020-07-01 17:14:04,088 DEBUG: fetched: [(0,)]
2020-07-01 17:14:04,146 ERROR: failed to import s3://some_bucket/some_target. You could also try downloading it manually, and adding it with `dvc add`. - Current operation was unsuccessful because 's3://some_bucket/some_target' requires existing cache on 's3' remote. See <https://man.dvc.org/config#cache> for information on how to set up remote cache.
------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/lib/python3.8/site-packages/dvc/command/imp_url.py", line 14, in run
    self.repo.imp_url(
  File "/usr/lib/python3.8/site-packages/dvc/repo/__init__.py", line 36, in wrapper
    ret = f(repo, *args, **kwargs)
  File "/usr/lib/python3.8/site-packages/dvc/repo/scm_context.py", line 4, in run
    result = method(repo, *args, **kw)
  File "/usr/lib/python3.8/site-packages/dvc/repo/imp_url.py", line 54, in imp_url
    stage.run()
  File "/home/anotherbugmaster/.local/lib/python3.8/site-packages/funcy/decorators.py", line 39, in wrapper
    return deco(call, *dargs, **dkwargs)
  File "/usr/lib/python3.8/site-packages/dvc/stage/decorators.py", line 35, in rwlocked
    return call()
  File "/home/anotherbugmaster/.local/lib/python3.8/site-packages/funcy/decorators.py", line 60, in __call__
    return self._func(*self._args, **self._kwargs)
  File "/usr/lib/python3.8/site-packages/dvc/stage/__init__.py", line 424, in run
    sync_import(self, dry, force)
  File "/usr/lib/python3.8/site-packages/dvc/stage/imports.py", line 29, in sync_import
    stage.save_deps()
  File "/usr/lib/python3.8/site-packages/dvc/stage/__init__.py", line 387, in save_deps
    dep.save()
  File "/usr/lib/python3.8/site-packages/dvc/output/base.py", line 268, in save
    self.info = self.save_info()
  File "/usr/lib/python3.8/site-packages/dvc/output/base.py", line 192, in save_info
    return self.remote.save_info(self.path_info)
  File "/usr/lib/python3.8/site-packages/dvc/remote/base.py", line 762, in save_info
    return self.tree.save_info(path_info, **kwargs)
  File "/usr/lib/python3.8/site-packages/dvc/remote/base.py", line 329, in save_info
    self.PARAM_CHECKSUM: self.get_hash(path_info, tree=tree, **kwargs)
  File "/usr/lib/python3.8/site-packages/dvc/remote/base.py", line 297, in get_hash
    hash_ = self.get_dir_hash(path_info, tree, **kwargs)
  File "/usr/lib/python3.8/site-packages/dvc/remote/base.py", line 311, in get_dir_hash
    raise RemoteCacheRequiredError(path_info)
dvc.exceptions.RemoteCacheRequiredError: Current operation was unsuccessful because 's3://some_bucket/some_target' requires existing cache on 's3' remote. See <https://man.dvc.org/config#cache> for information on how to set up remote cache.
------------------------------------------------------------

Having any troubles? Hit us up at https://dvc.org/support, we are always happy to help!
```
4. `dvc get-url s3://some_bucket/some_target -v`
```
2020-07-01 17:15:39,910 ERROR: unexpected error - 'NoneType' object has no attribute 'cache'
------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/lib/python3.8/site-packages/dvc/main.py", line 53, in main
    ret = cmd.run()
  File "/usr/lib/python3.8/site-packages/dvc/command/get_url.py", line 17, in run
    Repo.get_url(self.args.url, out=self.args.out)
  File "/usr/lib/python3.8/site-packages/dvc/repo/get_url.py", line 19, in get_url
    dep.save()
  File "/usr/lib/python3.8/site-packages/dvc/output/base.py", line 268, in save
    self.info = self.save_info()
  File "/usr/lib/python3.8/site-packages/dvc/output/base.py", line 192, in save_info
    return self.remote.save_info(self.path_info)
  File "/usr/lib/python3.8/site-packages/dvc/remote/base.py", line 762, in save_info
    return self.tree.save_info(path_info, **kwargs)
  File "/usr/lib/python3.8/site-packages/dvc/remote/base.py", line 329, in save_info
    self.PARAM_CHECKSUM: self.get_hash(path_info, tree=tree, **kwargs)
  File "/usr/lib/python3.8/site-packages/dvc/remote/base.py", line 297, in get_hash
    hash_ = self.get_dir_hash(path_info, tree, **kwargs)
  File "/usr/lib/python3.8/site-packages/dvc/remote/base.py", line 310, in get_dir_hash
    if not self.cache:
  File "/usr/lib/python3.8/site-packages/dvc/remote/base.py", line 184, in cache
    return getattr(self.repo.cache, self.scheme)
AttributeError: 'NoneType' object has no attribute 'cache'
------------------------------------------------------------

Having any troubles? Hit us up at https://dvc.org/support, we are always happy to help!
```

The same commands work with `https://some_domain/some_target` urls and I don't think that external cache were ever necessary to download files from S3.

### Please provide information about your setup

**Output of `dvc version`:**

```console
$ dvc version
1.1.2
```

**Additional Information (if any):**

If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.



Discussion:
Discord context https://discordapp.com/channels/485586884165107732/563406153334128681/727886431153946694
I found out a couple of things:
- `get-url` works in 0.93.0
- In order to make `import-url` work one need to set up s3 cache in _any_ bucket, not necessarily in the same bucket that contains the file that needs to be imported

That kind of solves the issue, but I don't get the logic behind this. Why would I need a cache in a separate bucket just to download the file from a completely different bucket? Seems weird because I need to download the file to my local machine anyway in order to compute hashes
@anotherbugmaster This is a well known bug that became more intrusive once we've adjusted the way we process inputs in get-url and import-url. It will be improved in the near future.

Resolving patch:
diff --git a/dvc/cache/base.py b/dvc/cache/base.py
--- a/dvc/cache/base.py
+++ b/dvc/cache/base.py
@@ -280,13 +280,14 @@ def _get_dir_info_hash(self, dir_info):
         return hash_info, to_info
 
     def save_dir_info(self, dir_info, hash_info=None):
-        if hash_info and not self.changed_cache_file(hash_info):
+        if (
+            hash_info
+            and hash_info.name == self.tree.PARAM_CHECKSUM
+            and not self.changed_cache_file(hash_info)
+        ):
             return hash_info
 
         hi, tmp_info = self._get_dir_info_hash(dir_info)
-        if hash_info:
-            assert hi == hash_info
-
         new_info = self.tree.hash_to_path_info(hi.value)
         if self.changed_cache_file(hi):
             self.tree.makedirs(new_info.parent)
diff --git a/dvc/tree/base.py b/dvc/tree/base.py
--- a/dvc/tree/base.py
+++ b/dvc/tree/base.py
@@ -7,11 +7,7 @@
 
 from funcy import cached_property
 
-from dvc.exceptions import (
-    DvcException,
-    DvcIgnoreInCollectedDirError,
-    RemoteCacheRequiredError,
-)
+from dvc.exceptions import DvcException, DvcIgnoreInCollectedDirError
 from dvc.hash_info import HashInfo
 from dvc.ignore import DvcIgnore
 from dvc.path_info import URLInfo
@@ -331,11 +327,8 @@ def _collect_dir(self, path_info, **kwargs):
         ]
 
     def get_dir_hash(self, path_info, **kwargs):
-        if not self.cache:
-            raise RemoteCacheRequiredError(path_info)
-
         dir_info = self._collect_dir(path_info, **kwargs)
-        return self.cache.save_dir_info(dir_info)
+        return self.repo.cache.local.save_dir_info(dir_info)
 
     def upload(self, from_info, to_info, name=None, no_progress_bar=False):
         if not hasattr(self, "_upload"):


Test patch:
diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py
--- a/tests/func/test_import_url.py
+++ b/tests/func/test_import_url.py
@@ -3,6 +3,7 @@
 
 import pytest
 
+from dvc.cache import Cache
 from dvc.dependency.base import DependencyDoesNotExistError
 from dvc.main import main
 from dvc.stage import Stage
@@ -141,23 +142,43 @@ def test_import_url(tmp_dir, dvc, workspace):
 
 
 @pytest.mark.parametrize(
-    "workspace",
+    "workspace, stage_md5, dir_md5",
     [
-        pytest.lazy_fixture("local_cloud"),
-        pytest.lazy_fixture("s3"),
-        pytest.lazy_fixture("gs"),
-        pytest.lazy_fixture("hdfs"),
+        (
+            pytest.lazy_fixture("local_cloud"),
+            "dc24e1271084ee317ac3c2656fb8812b",
+            "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
+        ),
+        (
+            pytest.lazy_fixture("s3"),
+            "2aa17f8daa26996b3f7a4cf8888ac9ac",
+            "ec602a6ba97b2dd07bd6d2cd89674a60.dir",
+        ),
+        (pytest.lazy_fixture("gs"), "fixme", "fixme",),
+        (
+            pytest.lazy_fixture("hdfs"),
+            "ec0943f83357f702033c98e70b853c8c",
+            "e6dcd267966dc628d732874f94ef4280.dir",
+        ),
         pytest.param(
             pytest.lazy_fixture("ssh"),
+            "dc24e1271084ee317ac3c2656fb8812b",
+            "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
             marks=pytest.mark.skipif(
                 os.name == "nt", reason="disabled on windows"
             ),
         ),
     ],
-    indirect=True,
+    indirect=["workspace"],
 )
-def test_import_url_dir(tmp_dir, dvc, workspace):
+def test_import_url_dir(tmp_dir, dvc, workspace, stage_md5, dir_md5):
     workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}})
+
+    # remove external cache to make sure that we don't need it to import dirs
+    with dvc.config.edit() as conf:
+        del conf["cache"]
+    dvc.cache = Cache(dvc)
+
     assert not (tmp_dir / "dir").exists()  # sanity check
     dvc.imp_url("remote://workspace/dir")
     assert set(os.listdir(tmp_dir / "dir")) == {"file", "subdir"}
@@ -165,4 +186,15 @@ def test_import_url_dir(tmp_dir, dvc, workspace):
     assert list(os.listdir(tmp_dir / "dir" / "subdir")) == ["subfile"]
     assert (tmp_dir / "dir" / "subdir" / "subfile").read_text() == "subfile"
 
+    assert (tmp_dir / "dir.dvc").read_text() == (
+        f"md5: {stage_md5}\n"
+        "frozen: true\n"
+        "deps:\n"
+        f"- md5: {dir_md5}\n"
+        "  path: remote://workspace/dir\n"
+        "outs:\n"
+        "- md5: b6dcab6ccd17ca0a8bf4a215a37d14cc.dir\n"
+        "  path: dir\n"
+    )
+
     assert dvc.status() == {}
diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py
--- a/tests/func/test_remote.py
+++ b/tests/func/test_remote.py
@@ -7,10 +7,10 @@
 from mock import patch
 
 from dvc.config import Config
-from dvc.exceptions import DownloadError, UploadError
+from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError
 from dvc.main import main
 from dvc.path_info import PathInfo
-from dvc.tree.base import BaseTree, RemoteCacheRequiredError
+from dvc.tree.base import BaseTree
 from dvc.tree.local import LocalTree
 from dvc.utils.fs import remove
 from tests.basic_env import TestDvc
@@ -262,7 +262,7 @@ def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory):
     with pytest.raises(RemoteCacheRequiredError):
         dvc.run(
             cmd="echo hello world",
-            deps=[os.fspath(external_dir)],
+            outs=[os.fspath(external_dir)],
             single_stage=True,
         )