Repository: iterative/dvc
Pull request #10072 (2023-11-08 09:48:51)

import/get: support pre-fetching LFS objects from Git-LFS repos
* [x] ❗ I have followed the [Contributing to DVC](https://dvc.org/doc/user-guide/contributing/core) checklist.

* [x] 📖 If this PR requires [documentation](https://dvc.org/doc) updates, I have created a separate PR (or issue, at least) in [dvc.org](https://github.com/iterative/dvc.org) and linked it here.

Thank you for the contribution - we'll try to review it as soon as possible. 🙏

Requires https://github.com/iterative/scmrepo/pull/278
Related: https://github.com/iterative/dvc/issues/9175

Example import from https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/tree/main/text_encoder:

[![asciicast](https://asciinema.org/a/FpkueOeDob9mdJ41Y8ByOW6Sx.svg)](https://asciinema.org/a/FpkueOeDob9mdJ41Y8ByOW6Sx)

LFS objects are only downloaded once (during the "Checking for LFS objects" step). The second "Downloading" message in the recording is the generic import/get status and is actually the reads from GitFilesystem (and not a network transfer).

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/iterative/dvc/pull/10072?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=iterative) Report
Attention: `4 lines` in your changes are missing coverage. Please review.
> Comparison is base [(`3d7bbb8`)](https://app.codecov.io/gh/iterative/dvc/commit/3d7bbb84922d6fc1622f860480d3cb09208e6708?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=iterative) 90.54% compared to head [(`81afb7a`)](https://app.codecov.io/gh/iterative/dvc/pull/10072?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=iterative) 90.13%.

| [Files](https://app.codecov.io/gh/iterative/dvc/pull/10072?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=iterative) | Patch % | Lines |
|---|---|---|
| [dvc/scm.py](https://app.codecov.io/gh/iterative/dvc/pull/10072?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=iterative#diff-ZHZjL3NjbS5weQ==) | 75.00% | [4 Missing :warning: ](https://app.codecov.io/gh/iterative/dvc/pull/10072?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=iterative) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main   #10072      +/-   ##
==========================================
- Coverage   90.54%   90.13%   -0.41%     
==========================================
  Files         487      487              
  Lines       37366    37385      +19     
  Branches     5432     5428       -4     
==========================================
- Hits        33833    33697     -136     
- Misses       2894     3004     +110     
- Partials      639      684      +45     
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/iterative/dvc/pull/10072?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=iterative).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=iterative).


Patch:
diff --git a/dvc/fs/__init__.py b/dvc/fs/__init__.py
index 77ac37cab..1e8627efc 100644
--- a/dvc/fs/__init__.py
+++ b/dvc/fs/__init__.py
@@ -5,6 +5,8 @@ from dvc_http import HTTPFileSystem, HTTPSFileSystem  # noqa: F401
 
 from dvc.config import ConfigError as RepoConfigError
 from dvc.config_schema import SCHEMA, Invalid
+
+# pylint: disable=unused-import
 from dvc_objects.fs import (  # noqa: F401
     LocalFileSystem,
     MemoryFileSystem,
@@ -27,7 +29,7 @@ from dvc_objects.fs.path import Path  # noqa: F401
 
 from .callbacks import Callback
 from .data import DataFileSystem  # noqa: F401
-from .dvc import DVCFileSystem  # noqa: F401
+from .dvc import DVCFileSystem
 from .git import GitFileSystem  # noqa: F401
 
 known_implementations.update(
@@ -47,6 +49,8 @@ known_implementations.update(
 def download(
     fs: "FileSystem", fs_path: str, to: str, jobs: Optional[int] = None
 ) -> int:
+    from dvc.scm import lfs_prefetch
+
     with Callback.as_tqdm_callback(
         desc=f"Downloading {fs.path.name(fs_path)}",
         unit="files",
@@ -70,6 +74,8 @@ def download(
             from_infos = [fs_path]
             to_infos = [to]
 
+        if isinstance(fs, DVCFileSystem):
+            lfs_prefetch(fs, from_infos)
         cb.set_size(len(from_infos))
         jobs = jobs or fs.jobs
         generic.copy(
diff --git a/dvc/scm.py b/dvc/scm.py
index 323adaf59..435b34a51 100644
--- a/dvc/scm.py
+++ b/dvc/scm.py
@@ -24,6 +24,8 @@ from dvc.progress import Tqdm
 if TYPE_CHECKING:
     from scmrepo.progress import GitProgressEvent
 
+    from dvc.fs import FileSystem
+
 
 class SCMError(DvcException):
     """Base class for source control management errors."""
@@ -262,3 +264,30 @@ def iter_revs(
 
     rev_resolver = partial(resolve_rev, scm)
     return group_by(rev_resolver, results)
+
+
+def lfs_prefetch(fs: "FileSystem", paths: List[str]):
+    from scmrepo.git.lfs import fetch as _lfs_fetch
+
+    from dvc.fs.dvc import DVCFileSystem
+    from dvc.fs.git import GitFileSystem
+
+    if isinstance(fs, DVCFileSystem) and isinstance(fs.repo.fs, GitFileSystem):
+        git_fs = fs.repo.fs
+        scm = fs.repo.scm
+        assert isinstance(scm, Git)
+    else:
+        return
+
+    try:
+        if "filter=lfs" not in git_fs.open(".gitattributes").read():
+            return
+    except OSError:
+        return
+    with TqdmGit(desc="Checking for Git-LFS objects") as pbar:
+        _lfs_fetch(
+            scm,
+            [git_fs.rev],
+            include=[(path if path.startswith("/") else f"/{path}") for path in paths],
+            progress=pbar.update_git,
+        )
diff --git a/pyproject.toml b/pyproject.toml
index 4fee24220..bc1194695 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -58,7 +58,7 @@ dependencies = [
     "requests>=2.22",
     "rich>=12",
     "ruamel.yaml>=0.17.11",
-    "scmrepo>=1.4.1,<2",
+    "scmrepo>=1.5.0,<2",
     "shortuuid>=0.5",
     "shtab<2,>=1.3.4",
     "tabulate>=0.8.7",


Test patch:
diff --git a/tests/func/test_scm.py b/tests/func/test_scm.py
index f7a227557..b5cab8aea 100644
--- a/tests/func/test_scm.py
+++ b/tests/func/test_scm.py
@@ -3,7 +3,7 @@ import os
 import pytest
 from git import Repo
 
-from dvc.scm import SCM, Git, NoSCM, SCMError
+from dvc.scm import SCM, Git, NoSCM, SCMError, lfs_prefetch
 
 
 def test_init_none(tmp_dir):
@@ -27,3 +27,20 @@ def test_init_sub_dir(tmp_dir):
 
     scm = SCM(os.fspath(subdir))
     assert scm.root_dir == os.fspath(tmp_dir)
+
+
+def test_lfs_prefetch(tmp_dir, dvc, scm, mocker):
+    mock_fetch = mocker.patch("scmrepo.git.lfs.fetch")
+    rev = scm.get_rev()
+
+    with dvc.switch(rev):
+        lfs_prefetch(dvc.dvcfs, ["foo"])
+        mock_fetch.assert_not_called()
+
+    tmp_dir.scm_gen(
+        ".gitattributes", ".lfs filter=lfs diff=lfs merge=lfs -text", commit="init lfs"
+    )
+    rev = scm.get_rev()
+    with dvc.switch(rev):
+        lfs_prefetch(dvc.dvcfs, ["foo"])
+        mock_fetch.assert_called_once()