Repository: iterative/dvc

Issue:
gc: fail to gc 2.x cache/remote objects
# Bug Report

## Description

`gc` does not garbage collect 2.x cache/remote objects

### Reproduce

```bash
set -x
# setup repos
REPO_DIR=$(mktemp -d)
REMOTE_DIR=$(mktemp -d)
cd $REPO_DIR
git init -q
dvc init -q
dvc remote add local $REMOTE_DIR
dvc remote default local
# add file to cache and push
echo foo >> foo
dvc add foo
dvc push
# update file and push
echo foo >> foo
dvc add foo
dvc push
# mimic old cache/remote and gc
cp -R .dvc/cache/files/md5 .dvc/cache
cp -R .dvc/cache/files/md5 $REMOTE_DIR
dvc gc -c -w 
tree $REPO_DIR/.dvc/cache
tree $REMOTE_DIR
```

### Expected

The output of the `tree` commands looks like:

```console
+ tree /var/folders/24/99_tf1xj3vx8k1k_jkdmnhq00000gn/T/tmp.A2IHOWGJ/.dvc/cache
/var/folders/24/99_tf1xj3vx8k1k_jkdmnhq00000gn/T/tmp.A2IHOWGJ/.dvc/cache
├── files
│   └── md5
│       ├── 5f
│       │   └── b7ba7e8447a836e774b66155f5776a
│       └── d3
└── md5
    ├── 5f
    │   └── b7ba7e8447a836e774b66155f5776a
    └── d3
        └── b07384d113edec49eaa6238ad5ff00

7 directories, 3 files
+ tree /var/folders/24/99_tf1xj3vx8k1k_jkdmnhq00000gn/T/tmp.LrAxuWxO
/var/folders/24/99_tf1xj3vx8k1k_jkdmnhq00000gn/T/tmp.LrAxuWxO
├── files
│   └── md5
│       ├── 5f
│       │   └── b7ba7e8447a836e774b66155f5776a
│       └── d3
└── md5
    ├── 5f
    │   └── b7ba7e8447a836e774b66155f5776a
    └── d3
        └── b07384d113edec49eaa6238ad5ff00
```

I would expect:

1. `d3/b07384d113edec49eaa6238ad5ff00` to be removed everywhere.
2. Non-3.x copies of `5f/b7ba7e8447a836e774b66155f5776a` to be removed.

### Environment information

```console
$ dvc doctor
DVC version: 3.19.1.dev20+gbf2af8639.d20230915
----------------------------------------------
Platform: Python 3.11.5 on macOS-13.5-arm64-arm-64bit
Subprojects:
        dvc_data = 2.16.2.dev1+g1672e69.d20230915
        dvc_objects = 1.0.1
        dvc_render = 0.5.3
        dvc_task = 0.3.0
        scmrepo = 1.3.1
Supports:
        azure (adlfs = 2023.8.0, knack = 0.11.0, azure-identity = 1.14.0),
        gdrive (pydrive2 = 1.17.0),
        gs (gcsfs = 2023.6.0),
        hdfs (fsspec = 2023.6.0, pyarrow = 13.0.0),
        http (aiohttp = 3.8.5, aiohttp-retry = 2.8.3),
        https (aiohttp = 3.8.5, aiohttp-retry = 2.8.3),
        oss (ossfs = 2023.8.0),
        s3 (s3fs = 2023.6.0, boto3 = 1.28.17),
        ssh (sshfs = 2023.7.0),
        webdav (webdav4 = 0.9.8),
        webdavs (webdav4 = 0.9.8),
        webhdfs (fsspec = 2023.6.0)
Config:
        Global: /Users/dave/Library/Application Support/dvc
        System: /Library/Application Support/dvc
```


Discussion:
Marking as a p0 since it seems to break the ability to gc after 3.0 migration.
Just to clarify, this is reproducible but the repro script should contain 

```
cp -R .dvc/cache/files/md5/* .dvc/cache
cp -R .dvc/cache/files/md5/* $REMOTE_DIR
```

(the original script ends up copying into `.dvc/cache/md5/...` instead of `.dvc/cache/...` when trying to populate the legacy cache location)
Two issues here:

1. We were not checking hashname in dvc-data when GC'ing, so in cases where a 2.x and 3.x hash had the same hash string both 2.x and 3.x versions would be preserved (this affected it both ways where 2.x objects could be preserved when we only wanted to keep the 3.x version, and vice-versa)
2. DVC `gc -c` logic would only call gc for the remotes which contained the used objects we wanted to preserve (so in the event we only have 3.x objects we would not try to gc the 2.x remote location). In cases where we had a mix of 3.x and 2.x used objects to preserve `gc -c` would behave as expected (except for the 2.x/3.x hash overlap case from `1.`)

Resolving patch:
diff --git a/dvc/output.py b/dvc/output.py
--- a/dvc/output.py
+++ b/dvc/output.py
@@ -1160,10 +1160,11 @@ def _collect_used_dir_cache(
             return obj.filter(prefix)
         return obj
 
-    def get_used_objs(  # noqa: C901
+    def get_used_objs(  # noqa: C901, PLR0911
         self, **kwargs
     ) -> Dict[Optional["HashFileDB"], Set["HashInfo"]]:
         """Return filtered set of used object IDs for this out."""
+        from dvc.cachemgr import LEGACY_HASH_NAMES
 
         if not self.use_cache:
             return {}
@@ -1202,13 +1203,17 @@ def get_used_objs(  # noqa: C901
             return {}
 
         if self.remote:
-            remote = self.repo.cloud.get_remote_odb(
+            remote_odb = self.repo.cloud.get_remote_odb(
                 name=self.remote, hash_name=self.hash_name
             )
-        else:
-            remote = None
-
-        return {remote: self._named_obj_ids(obj)}
+            other_odb = self.repo.cloud.get_remote_odb(
+                name=self.remote,
+                hash_name=(
+                    "md5" if self.hash_name in LEGACY_HASH_NAMES else "md5-dos2unix"
+                ),
+            )
+            return {remote_odb: self._named_obj_ids(obj), other_odb: set()}
+        return {None: self._named_obj_ids(obj)}
 
     def _named_obj_ids(self, obj):
         name = str(self)
diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py
--- a/dvc/repo/gc.py
+++ b/dvc/repo/gc.py
@@ -1,5 +1,5 @@
 import logging
-from typing import TYPE_CHECKING, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, List, Optional
 
 from dvc.exceptions import InvalidArgumentError
 
@@ -7,8 +7,7 @@
 
 if TYPE_CHECKING:
     from dvc.repo import Repo
-    from dvc_data.hashfile.db import HashFileDB
-    from dvc_data.hashfile.hash_info import HashInfo
+    from dvc.repo.index import ObjectContainer
 
 logger = logging.getLogger(__name__)
 
@@ -33,22 +32,25 @@ def _validate_args(**kwargs):
         raise InvalidArgumentError("`--num` can only be used alongside `--rev`")
 
 
-def _used_obj_ids_not_in_remote(repo, default_remote, jobs, remote_odb_to_obj_ids):
+def _used_obj_ids_not_in_remote(
+    remote_odb_to_obj_ids: "ObjectContainer", jobs: Optional[int] = None
+):
     used_obj_ids = set()
     remote_oids = set()
-    for remote_odb, obj_ids in remote_odb_to_obj_ids:
-        if remote_odb is None:
-            remote_odb = repo.cloud.get_remote_odb(default_remote, "gc --not-in-remote")
-
+    for remote_odb, obj_ids in remote_odb_to_obj_ids.items():
+        assert remote_odb
         remote_oids.update(
-            remote_odb.list_oids_exists({x.value for x in obj_ids}, jobs=jobs)
+            remote_odb.list_oids_exists(
+                {x.value for x in obj_ids if x.value},
+                jobs=jobs,
+            )
         )
         used_obj_ids.update(obj_ids)
     return {obj for obj in used_obj_ids if obj.value not in remote_oids}
 
 
 @locked
-def gc(  # noqa: PLR0912, PLR0913, C901
+def gc(  # noqa: PLR0913, C901
     self: "Repo",
     all_branches: bool = False,
     cloud: bool = False,
@@ -92,8 +94,7 @@ def gc(  # noqa: PLR0912, PLR0913, C901
         repos = []
     all_repos = [Repo(path) for path in repos]
 
-    odb_to_obj_ids: List[Tuple[Optional["HashFileDB"], Set["HashInfo"]]] = []
-
+    odb_to_obj_ids: "ObjectContainer" = {}
     with ExitStack() as stack:
         for repo in all_repos:
             stack.enter_context(repo.lock)
@@ -112,17 +113,17 @@ def gc(  # noqa: PLR0912, PLR0913, C901
                 revs=[rev] if rev else None,
                 num=num or 1,
             ).items():
-                odb_to_obj_ids.append((odb, obj_ids))
-
-    if not odb_to_obj_ids:
-        odb_to_obj_ids = [(None, set())]
+                if odb not in odb_to_obj_ids:
+                    odb_to_obj_ids[odb] = set()
+                odb_to_obj_ids[odb].update(obj_ids)
 
+    if cloud or not_in_remote:
+        _merge_remote_obj_ids(self, remote, odb_to_obj_ids)
     if not_in_remote:
-        used_obj_ids = _used_obj_ids_not_in_remote(self, remote, jobs, odb_to_obj_ids)
+        used_obj_ids = _used_obj_ids_not_in_remote(odb_to_obj_ids, jobs=jobs)
     else:
         used_obj_ids = set()
-        for _, obj_ids in odb_to_obj_ids:
-            used_obj_ids.update(obj_ids)
+        used_obj_ids.update(*odb_to_obj_ids.values())
 
     for scheme, odb in self.cache.by_scheme():
         if not odb:
@@ -134,11 +135,25 @@ def gc(  # noqa: PLR0912, PLR0913, C901
     if not cloud:
         return
 
-    for remote_odb, obj_ids in odb_to_obj_ids:
-        if remote_odb is None:
-            remote_odb = repo.cloud.get_remote_odb(remote, "gc -c")
+    for remote_odb, obj_ids in odb_to_obj_ids.items():
+        assert remote_odb is not None
         removed = ogc(remote_odb, obj_ids, jobs=jobs)
         if removed:
             get_index(remote_odb).clear()
         else:
             logger.info("No unused cache to remove from remote.")
+
+
+def _merge_remote_obj_ids(
+    repo: "Repo", remote: Optional[str], used_objs: "ObjectContainer"
+):
+    # Merge default remote used objects with remote-per-output used objects
+    default_obj_ids = used_objs.pop(None, set())
+    remote_odb = repo.cloud.get_remote_odb(remote, "gc -c", hash_name="md5")
+    if remote_odb not in used_objs:
+        used_objs[remote_odb] = set()
+    used_objs[remote_odb].update(default_obj_ids)
+    legacy_odb = repo.cloud.get_remote_odb(remote, "gc -c", hash_name="md5-dos2unix")
+    if legacy_odb not in used_objs:
+        used_objs[legacy_odb] = set()
+    used_objs[legacy_odb].update(default_obj_ids)


Test patch: