Repository: iterative/dvc

Issue:
cloud versioning: status command
`dvc status` or `dvc data status` should work with cloud-versioned remotes.

Edit: The scope for this ticket is limited to `version_aware` remotes.


Discussion:
I'd rather have it in `dvc data status`, but there's not any support for cloud status in that command yet, so it depends on how much effort this would be
For the record: looking at it closer, `dvc data status` is closer to the spirit, but `dvc status` is already a trainwreck and it is easy to shove another cloud-related functionality there, but that introduces a yet another type of output and users won't be happy, esp in a newly released feature. For now trying to put it into `dvc data status` in an elegant way 🙂 
Looks like https://github.com/iterative/dvc-data/issues/208 is also needed here to make data status use index comfortably, but trying to salvage old state db for now...
We have `dvc data status` use indexes now #8846 

On second thought, I am wondering if we are making a similar mistake to `dvc status -c` back in a day by trying to shove something that doesn't really belong to `status`. Status is about your local workspace, but for cloud versioning we are using a remote workspace, so I'm wondering if what we actually want is something closer to `diff` rather than to status. For example, in git you can diff against a remote, but you can't status against it. Which makes sense, since `status` is about `index vs workspace`.

So if this should really be `dvc data diff`, it is still interesting how to designate our remote vs git revs. Maybe `dvc diff remote://mycloudversioningremote`? Or `dvc/mycloudversioningremote` or just some special `--worktree` flag or something? 🤔 


Patch:
diff --git a/dvc/commands/data.py b/dvc/commands/data.py
--- a/dvc/commands/data.py
+++ b/dvc/commands/data.py
@@ -21,12 +21,14 @@
 
 class CmdDataStatus(CmdBase):
     COLORS = {
+        "not_in_remote": "red",
         "not_in_cache": "red",
         "committed": "green",
         "uncommitted": "yellow",
         "untracked": "cyan",
     }
     LABELS = {
+        "not_in_remote": "Not in remote",
         "not_in_cache": "Not in cache",
         "committed": "DVC committed changes",
         "uncommitted": "DVC uncommitted changes",
@@ -34,6 +36,7 @@ class CmdDataStatus(CmdBase):
         "unchanged": "DVC unchanged files",
     }
     HINTS = {
+        "not_in_remote": ('use "dvc push <file>..." to upload files',),
         "not_in_cache": ('use "dvc fetch <file>..." to download files',),
         "committed": ("git commit the corresponding dvc files to update the repo",),
         "uncommitted": (
@@ -114,6 +117,7 @@ def run(self) -> int:
             status = self.repo.data_status(
                 granular=self.args.granular,
                 untracked_files=self.args.untracked_files,
+                not_in_remote=self.args.not_in_remote,
             )
 
         if not self.args.unchanged:
@@ -239,6 +243,12 @@ def add_parser(subparsers, parent_parser):
         nargs="?",
         help="Show untracked files.",
     )
+    data_status_parser.add_argument(
+        "--not-in-remote",
+        action="store_true",
+        default=False,
+        help="Show files missing from remote.",
+    )
     data_status_parser.set_defaults(func=CmdDataStatus)
 
     DATA_LS_HELP = "List data tracked by DVC with its metadata."
diff --git a/dvc/repo/data.py b/dvc/repo/data.py
--- a/dvc/repo/data.py
+++ b/dvc/repo/data.py
@@ -58,8 +58,10 @@ def _diff(
     new: "DataIndex",
     *,
     granular: bool = False,
-    with_missing: bool = False,
+    not_in_cache: bool = False,
+    not_in_remote: bool = False,
 ) -> Dict[str, List[str]]:
+    from dvc_data.index import StorageError
     from dvc_data.index.diff import UNCHANGED, UNKNOWN, diff
 
     ret: Dict[str, List[str]] = {}
@@ -94,7 +96,7 @@ def _add_change(typ, change):
             continue
 
         if (
-            with_missing
+            not_in_cache
             and change.old
             and change.old.hash_info
             and not old.storage_map.cache_exists(change.old)
@@ -102,6 +104,17 @@ def _add_change(typ, change):
             # NOTE: emulating previous behaviour
             _add_change("not_in_cache", change)
 
+        try:
+            if (
+                not_in_remote
+                and change.old
+                and change.old.hash_info
+                and not old.storage_map.remote_exists(change.old)
+            ):
+                _add_change("not_in_remote", change)
+        except StorageError:
+            pass
+
         _add_change(change.typ, change)
 
     return ret
@@ -155,7 +168,7 @@ def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
         return _diff(
             repo.index.data["repo"],
             workspace,
-            with_missing=True,
+            not_in_cache=True,
             **kwargs,
         )
 
@@ -177,6 +190,7 @@ def _diff_head_to_index(
 
 class Status(TypedDict):
     not_in_cache: List[str]
+    not_in_remote: List[str]
     committed: Dict[str, List[str]]
     uncommitted: Dict[str, List[str]]
     untracked: List[str]
@@ -203,12 +217,20 @@ def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]
     return [repo.fs.path.relpath(file, start) for file in files]
 
 
-def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status:
+def status(
+    repo: "Repo",
+    untracked_files: str = "no",
+    not_in_remote: bool = False,
+    **kwargs: Any,
+) -> Status:
     from dvc.scm import NoSCMError, SCMError
 
     head = kwargs.pop("head", "HEAD")
-    uncommitted_diff = _diff_index_to_wtree(repo, **kwargs)
-    not_in_cache = uncommitted_diff.pop("not_in_cache", [])
+    uncommitted_diff = _diff_index_to_wtree(
+        repo,
+        not_in_remote=not_in_remote,
+        **kwargs,
+    )
     unchanged = set(uncommitted_diff.pop("unchanged", []))
 
     try:
@@ -223,7 +245,8 @@ def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status:
     untracked = _transform_git_paths_to_dvc(repo, untracked)
     # order matters here
     return Status(
-        not_in_cache=not_in_cache,
+        not_in_cache=uncommitted_diff.pop("not_in_cache", []),
+        not_in_remote=uncommitted_diff.pop("not_in_remote", []),
         committed=committed_diff,
         uncommitted=uncommitted_diff,
         untracked=untracked,


Test patch:
diff --git a/tests/func/test_data_status.py b/tests/func/test_data_status.py
--- a/tests/func/test_data_status.py
+++ b/tests/func/test_data_status.py
@@ -13,6 +13,7 @@
     "uncommitted": {},
     "git": {},
     "not_in_cache": [],
+    "not_in_remote": [],
     "unchanged": [],
     "untracked": [],
 }
@@ -81,6 +82,7 @@ def test_directory(M, tmp_dir, dvc, scm):
     }
 
     assert dvc.data_status(granular=True, untracked_files="all") == {
+        **EMPTY_STATUS,
         "committed": {
             "added": M.unordered(
                 join("dir", "bar"),
@@ -396,6 +398,42 @@ def test_missing_dir_object_from_index(M, tmp_dir, dvc, scm):
     }
 
 
+def test_missing_remote_cache(M, tmp_dir, dvc, scm, local_remote):
+    tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}})
+    tmp_dir.dvc_gen("foobar", "foobar")
+
+    assert dvc.data_status(untracked_files="all", not_in_remote=True) == {
+        **EMPTY_STATUS,
+        "untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"),
+        "committed": {"added": M.unordered("foobar", join("dir", ""))},
+        "not_in_remote": M.unordered("foobar", join("dir", "")),
+        "git": M.dict(),
+    }
+
+    assert dvc.data_status(
+        granular=True, untracked_files="all", not_in_remote=True
+    ) == {
+        **EMPTY_STATUS,
+        "untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"),
+        "committed": {
+            "added": M.unordered(
+                "foobar",
+                join("dir", ""),
+                join("dir", "foo"),
+                join("dir", "bar"),
+            )
+        },
+        "uncommitted": {},
+        "not_in_remote": M.unordered(
+            "foobar",
+            join("dir", ""),
+            join("dir", "foo"),
+            join("dir", "bar"),
+        ),
+        "git": M.dict(),
+    }
+
+
 def test_root_from_dir_to_file(M, tmp_dir, dvc, scm):
     tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}})
     remove("data")
diff --git a/tests/unit/command/test_data_status.py b/tests/unit/command/test_data_status.py
--- a/tests/unit/command/test_data_status.py
+++ b/tests/unit/command/test_data_status.py
@@ -51,6 +51,7 @@ def test_cli(dvc, mocker, mocked_status):
     assert cmd.run() == 0
     status.assert_called_once_with(
         untracked_files="all",
+        not_in_remote=False,
         granular=True,
     )