Repository: iterative/dvc

Issue:
push: fails with an HTTP external URL tracked by a .dvc file
# Bug Report

<!--
## Issue name

Issue names must follow the pattern `command: description` where the command is the dvc command that you are trying to run. The description should describe the consequence of the bug.

Example: `repro: doesn't detect input changes`
-->

## Description

I have a project that has external data files added with `import-url`. With the latest DVC (3.28), when I run `dvc push`, it fails with an unsupported operation in the HTTP remote:

<details>
<summary>Backtrace</summary>
<pre>
2023-11-04 15:21:53,740 ERROR: unexpected error
Traceback (most recent call last):
  File "/home/mde48/miniforge3/envs/lkpm/lib/python3.10/site-packages/dvc/cli/__init__.py", line 209, in main
    ret = cmd.do_run()
  File "/home/mde48/miniforge3/envs/lkpm/lib/python3.10/site-packages/dvc/cli/command.py", line 26, in do_run
    return self.run()
  File "/home/mde48/miniforge3/envs/lkpm/lib/python3.10/site-packages/dvc/commands/data_sync.py", line 64, in run
    processed_files_count = self.repo.push(
  File "/home/mde48/miniforge3/envs/lkpm/lib/python3.10/site-packages/dvc/repo/__init__.py", line 61, in wrapper
    return f(repo, *args, **kwargs)
  File "/home/mde48/miniforge3/envs/lkpm/lib/python3.10/site-packages/dvc/repo/push.py", line 115, in push
    push_transferred, push_failed = ipush(
  File "/home/mde48/miniforge3/envs/lkpm/lib/python3.10/site-packages/dvc_data/index/push.py", line 59, in push
    old = build(data.path, data.fs)
  File "/home/mde48/miniforge3/envs/lkpm/lib/python3.10/site-packages/dvc_data/index/build.py", line 80, in build
    for entry in build_entries(path, fs, ignore=ignore):
  File "/home/mde48/miniforge3/envs/lkpm/lib/python3.10/site-packages/dvc_data/index/build.py", line 52, in build_entries
    walk_iter = fs.walk(path)
  File "/home/mde48/miniforge3/envs/lkpm/lib/python3.10/site-packages/dvc_http/__init__.py", line 153, in walk
    raise NotImplementedError
NotImplementedError
</pre>
</details>

### Reproduce

1. dvc init
2. dvc import-url https://files.grouplens.org/datasets/movielens/ml-latest-small.zip
3. dvc stage add -n extract -d ml-latest-small.zip -o ml-latest-small unzip ml-latest-small.zip
4. dvc repro
5. dvc remote add test ../test-remote
6. dvc push -r test

### Expected

The cached outputs (and maybe a copy of the input, but I think that's a separate issue) to be pushed to the remote.

### Environment information

**Output of `dvc doctor`:**

```console
$ dvc doctor
DVC version: 3.28.0 (conda)
---------------------------
Platform: Python 3.10.13 on Linux-5.15.0-88-generic-x86_64-with-glibc2.35
Subprojects:
        dvc_data = 2.20.0
        dvc_objects = 1.1.0
        dvc_render = 0.6.0
        dvc_task = 0.3.0
        scmrepo = 1.4.1
Supports:
        http (aiohttp = 3.8.6, aiohttp-retry = 2.8.3),
        https (aiohttp = 3.8.6, aiohttp-retry = 2.8.3)
Config:
        Global: /home/mde48/.config/dvc
        System: /etc/xdg/dvc
Cache types: hardlink, symlink
Cache directory: ext4 on /dev/mapper/ubuntu--vg-ubuntu--lv
Caches: local
Remotes: local
Workspace directory: ext4 on /dev/mapper/ubuntu--vg-ubuntu--lv
Repo: dvc, git
Repo.site_cache_dir: /var/tmp/dvc/repo/45dbcc8b97aba74d36934a2f402ce97a
```



Discussion:
Can confirm that I'm able to reproduce. Seems like this works fine in 3.27.0 and was introduced in 3.27.1. I'm guessing it's related to https://github.com/iterative/dvc/pull/9807 (cc @efiop).

Resolving patch:
diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py
--- a/dvc/repo/fetch.py
+++ b/dvc/repo/fetch.py
@@ -1,4 +1,4 @@
-from typing import List, Tuple
+from typing import TYPE_CHECKING, List, Tuple
 
 from dvc.exceptions import DownloadError
 from dvc.log import logger
@@ -7,6 +7,10 @@
 
 from . import locked
 
+if TYPE_CHECKING:
+    from dvc.output import Output
+    from dvc.stage import Stage
+
 logger = logger.getChild(__name__)
 
 
@@ -32,6 +36,7 @@ def _collect_indexes(  # noqa: PLR0913
     types=None,
     config=None,
     onerror=None,
+    push=False,
 ):
     indexes = {}
     collection_exc = None
@@ -42,6 +47,16 @@ def _collect_indexes(  # noqa: PLR0913
         core["remote"] = remote
         config["core"] = core
 
+    def stage_filter(stage: "Stage") -> bool:
+        if push and stage.is_repo_import:
+            return False
+        return True
+
+    def outs_filter(out: "Output") -> bool:
+        if push and not out.can_push:
+            return False
+        return True
+
     for rev in repo.brancher(
         revs=revs,
         all_branches=all_branches,
@@ -57,6 +72,8 @@ def _collect_indexes(  # noqa: PLR0913
                 recursive=recursive,
                 max_size=max_size,
                 types=types,
+                stage_filter=stage_filter,
+                outs_filter=outs_filter,
             )
 
             idx.data["repo"].onerror = _make_index_onerror(onerror, rev)
diff --git a/dvc/repo/push.py b/dvc/repo/push.py
--- a/dvc/repo/push.py
+++ b/dvc/repo/push.py
@@ -87,6 +87,7 @@ def push(  # noqa: PLR0913
         recursive=recursive,
         all_commits=all_commits,
         revs=revs,
+        push=True,
     )
 
     cache_key = (
@@ -107,6 +108,7 @@ def push(  # noqa: PLR0913
             push=True,
         )
 
+    push_transferred, push_failed = 0, 0
     try:
         with ui.progress(
             desc="Pushing",
@@ -131,6 +133,11 @@ def push(  # noqa: PLR0913
         for fs_index in data:
             fs_index.close()
 
+        if push_transferred:
+            # NOTE: dropping cached index to force reloading from newly saved
+            # metadata from version-aware remotes
+            self.drop_data_index()
+
     transferred_count += push_transferred
     failed_count += push_failed
     if failed_count:


Test patch:
diff --git a/dvc/testing/remote_tests.py b/dvc/testing/remote_tests.py
--- a/dvc/testing/remote_tests.py
+++ b/dvc/testing/remote_tests.py
@@ -172,6 +172,8 @@ def test_file(self, tmp_dir, dvc, run_copy, remote_version_aware):
         run_copy("foo", "foo_copy", name="copy")
 
         dvc.push()
+        assert (remote_version_aware / "foo").read_text() == "foo"
+        assert (remote_version_aware / "foo_copy").read_text() == "foo"
         foo_dvc = (tmp_dir / "foo.dvc").read_text()
         assert "version_id" in foo_dvc
         stage = stage.reload()
@@ -181,18 +183,24 @@ def test_file(self, tmp_dir, dvc, run_copy, remote_version_aware):
 
         remove(dvc.cache.local.path)
         remove(tmp_dir / "foo")
+        remove(tmp_dir / "foo_copy")
 
         dvc.pull()
         assert (tmp_dir / "foo").read_text() == "foo"
+        assert (tmp_dir / "foo_copy").read_text() == "foo"
         assert (tmp_dir / "foo.dvc").read_text() == foo_dvc
         assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
 
         dvc.push()
+        assert (remote_version_aware / "foo").read_text() == "foo"
+        assert (remote_version_aware / "foo_copy").read_text() == "foo"
         assert (tmp_dir / "foo.dvc").read_text() == foo_dvc
         assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
 
         dvc.reproduce()
         dvc.push()
+        assert (remote_version_aware / "foo").read_text() == "foo"
+        assert (remote_version_aware / "foo_copy").read_text() == "foo"
         assert (tmp_dir / "foo.dvc").read_text() == foo_dvc
         assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
 
@@ -241,6 +249,12 @@ def test_dir(self, tmp_dir, dvc, run_copy, remote_version_aware):
         assert (tmp_dir / "data_dir.dvc").read_text() == data_dir_dvc
         assert (tmp_dir / "dvc.lock").read_text() == dvc_lock
 
+        dvc.cache.local.clear()
+        remove(tmp_dir / "data_dir")
+        dvc.push()
+        assert (remote_version_aware / "data_dir").exists()
+        assert (remote_version_aware / "data_dir" / "data").exists()
+
 
 class TestRemoteWorktree:
     def test_file(self, tmp_dir, dvc, remote_worktree):
diff --git a/tests/func/api/test_data.py b/tests/func/api/test_data.py
--- a/tests/func/api/test_data.py
+++ b/tests/func/api/test_data.py
@@ -41,7 +41,7 @@ def test_open_external(tmp_dir, erepo_dir, cloud):
             # NOTE: need file to be other size for Mac
             erepo_dir.dvc_gen("version", "branchver", commit="add version")
 
-    erepo_dir.dvc.push(all_branches=True)
+    assert erepo_dir.dvc.push(all_branches=True) == 2
 
     # Remove cache to force download
     remove(erepo_dir.dvc.cache.local.path)
diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py
--- a/tests/func/test_data_cloud.py
+++ b/tests/func/test_data_cloud.py
@@ -486,7 +486,7 @@ def test_pull_partial(tmp_dir, dvc, local_remote):
     clean(["foo"], dvc)
 
     stats = dvc.pull(os.path.join("foo", "bar"))
-    assert stats["fetched"] == 2
+    assert stats["fetched"] == 3
     assert (tmp_dir / "foo").read_text() == {"bar": {"baz": "baz"}}
 
 
diff --git a/tests/func/test_virtual_directory.py b/tests/func/test_virtual_directory.py
--- a/tests/func/test_virtual_directory.py
+++ b/tests/func/test_virtual_directory.py
@@ -182,7 +182,7 @@ def test_partial_checkout_and_update(M, tmp_dir, dvc, remote):
 
     assert dvc.pull("dir/subdir") == M.dict(
         added=[join("dir", "")],
-        fetched=2,
+        fetched=3,
     )
     assert (tmp_dir / "dir").read_text() == {"subdir": {"lorem": "lorem"}}