Repository: iterative/dvc

Issue:
Command to extract/show this metadata



Comments:

Patch:
diff --git a/dvc/cli/actions.py b/dvc/cli/actions.py
--- a/dvc/cli/actions.py
+++ b/dvc/cli/actions.py
@@ -1,6 +1,15 @@
 from argparse import _AppendAction
 
 
+class CommaSeparatedArgs(_AppendAction):  # pylint: disable=protected-access
+    def __call__(self, parser, namespace, values, option_string=None):
+        from funcy import ldistinct
+
+        items = getattr(namespace, self.dest) or []
+        items.extend(map(str.strip, values.split(",")))
+        setattr(namespace, self.dest, ldistinct(items))
+
+
 class KeyValueArgs(_AppendAction):
     def __call__(self, parser, namespace, values, option_string=None):
         items = getattr(namespace, self.dest) or {}
diff --git a/dvc/commands/data.py b/dvc/commands/data.py
--- a/dvc/commands/data.py
+++ b/dvc/commands/data.py
@@ -1,9 +1,12 @@
 import argparse
 import logging
-from typing import TYPE_CHECKING
+from operator import itemgetter
+from typing import TYPE_CHECKING, Any, Dict, Iterable, Set
 
 from funcy import chunks, compact, log_durations
 
+from dvc.cli import completion
+from dvc.cli.actions import CommaSeparatedArgs
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link, fix_subparsers
 from dvc.ui import ui
@@ -124,6 +127,65 @@ def run(self) -> int:
         return self._show_status(status)
 
 
+class CmdDataLs(CmdBase):
+    @staticmethod
+    def _show_table(
+        d: Iterable[Dict[str, Any]],
+        filter_types: Set[str],
+        filter_labels: Set[str],
+        markdown: bool = False,
+    ) -> None:
+        from rich.style import Style
+
+        from dvc.compare import TabularData
+
+        td = TabularData(
+            columns=["Path", "Type", "Labels", "Description"], fill_value="-"
+        )
+        for entry in sorted(d, key=itemgetter("path")):
+            typ = entry.get("type", "")
+            desc = entry.get("desc", "-")
+            labels = entry.get("labels", [])
+
+            if filter_types and typ not in filter_types:
+                continue
+            if filter_labels and filter_labels.isdisjoint(labels):
+                continue
+
+            rich_label = ui.rich_text()
+            for index, label in enumerate(labels):
+                if index:
+                    rich_label.append(",")
+                style = Style(bold=label in filter_labels, color="green")
+                rich_label.append(label, style=style)
+
+            if markdown and desc:
+                desc = desc.partition("\n")[0]
+
+            path = ui.rich_text(entry["path"], style="cyan")
+            type_style = Style(bold=typ in filter_types, color="yellow")
+            typ = ui.rich_text(entry.get("type", ""), style=type_style)
+            td.append([path, typ or "-", rich_label or "-", desc])
+
+        td.render(markdown=markdown, rich_table=True)
+
+    def run(self):
+        from dvc.repo.data import ls
+
+        filter_labels = set(self.args.labels)
+        filter_types = set(self.args.type)
+        d = ls(
+            self.repo, targets=self.args.targets, recursive=self.args.recursive
+        )
+        self._show_table(
+            d,
+            filter_labels=filter_labels,
+            filter_types=filter_types,
+            markdown=self.args.markdown,
+        )
+        return 0
+
+
 def add_parser(subparsers, parent_parser):
     data_parser = subparsers.add_parser(
         "data",
@@ -181,3 +243,51 @@ def add_parser(subparsers, parent_parser):
         help="Show untracked files.",
     )
     data_status_parser.set_defaults(func=CmdDataStatus)
+
+    DATA_LS_HELP = "List data tracked by DVC with its metadata."
+    data_ls_parser = data_subparsers.add_parser(
+        "ls",
+        aliases=["list"],
+        parents=[parent_parser],
+        description=append_doc_link(DATA_LS_HELP, "data/ls"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        help=DATA_LS_HELP,
+    )
+    data_ls_parser.add_argument(
+        "--md",
+        "--show-md",
+        dest="markdown",
+        action="store_true",
+        default=False,
+        help="Show tabulated output in the Markdown format (GFM).",
+    )
+    data_ls_parser.add_argument(
+        "--type",
+        action=CommaSeparatedArgs,
+        default=[],
+        help="Comma-separated list of type to filter.",
+    )
+    data_ls_parser.add_argument(
+        "--labels",
+        action=CommaSeparatedArgs,
+        default=[],
+        help="Comma-separated list of labels to filter.",
+    )
+    data_ls_parser.add_argument(
+        "-R",
+        "--recursive",
+        action="store_true",
+        default=False,
+        help="Recursively list from the specified directories.",
+    )
+    data_ls_parser.add_argument(
+        "targets",
+        default=None,
+        nargs="*",
+        help=(
+            "Limit command scope to these tracked files/directories, "
+            ".dvc files, or stage names."
+        ),
+    ).complete = completion.DVCFILES_AND_STAGE
+
+    data_ls_parser.set_defaults(func=CmdDataLs)
diff --git a/dvc/repo/data.py b/dvc/repo/data.py
--- a/dvc/repo/data.py
+++ b/dvc/repo/data.py
@@ -1,11 +1,13 @@
 import os
 import posixpath
 from collections import defaultdict
+from itertools import chain
 from typing import (
     TYPE_CHECKING,
     Any,
     Dict,
     Iterable,
+    Iterator,
     List,
     Optional,
     TypedDict,
@@ -300,3 +302,18 @@ def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status:
         unchanged=list(unchanged),
         git=git_info,
     )
+
+
+def ls(
+    repo: "Repo",
+    targets: List[Optional[str]] = None,
+    recursive: bool = False,
+) -> Iterator[Dict[str, Any]]:
+    targets = targets or [None]
+    pairs = chain.from_iterable(
+        repo.stage.collect_granular(target, recursive=recursive)
+        for target in targets
+    )
+    for stage, filter_info in pairs:
+        for out in stage.filter_outs(filter_info):
+            yield {"path": str(out), **out.annot.to_dict()}


Test patch:
diff --git a/tests/func/test_data_ls.py b/tests/func/test_data_ls.py
new file mode 100644
--- /dev/null
+++ b/tests/func/test_data_ls.py
@@ -0,0 +1,31 @@
+from operator import itemgetter
+
+from dvc.repo.data import ls
+
+
+def test_data_ls(tmp_dir, dvc):
+    assert not list(ls(dvc))
+
+    tmp_dir.dvc_gen("bar", "bar")
+    tmp_dir.gen("foo", "foo")
+    dvc.add(
+        "foo",
+        meta={"key": "value"},
+        labels=["l1", "l2"],
+        type="t1",
+        desc="foo",
+    )
+
+    foo_entry = {
+        "path": "foo",
+        "desc": "foo",
+        "type": "t1",
+        "labels": ["l1", "l2"],
+        "meta": {"key": "value"},
+    }
+    assert sorted(ls(dvc), key=itemgetter("path")) == [
+        {"path": "bar"},
+        foo_entry,
+    ]
+    assert list(ls(dvc, targets=["foo"])) == [foo_entry]
+    assert list(ls(dvc, targets=["foo"], recursive=True)) == [foo_entry]
diff --git a/tests/unit/command/test_compat_flag.py b/tests/unit/command/test_compat_flag.py
--- a/tests/unit/command/test_compat_flag.py
+++ b/tests/unit/command/test_compat_flag.py
@@ -15,6 +15,7 @@ def _id_gen(val) -> str:
     "args, key",
     [
         (["dag", "--show-md"], "markdown"),
+        (["data", "ls", "--show-md"], "markdown"),
         (["diff", "--show-json"], "json"),
         (["diff", "--show-md"], "markdown"),
         (["experiments", "diff", "--show-json"], "json"),
diff --git a/tests/unit/command/test_data_ls.py b/tests/unit/command/test_data_ls.py
new file mode 100644
--- /dev/null
+++ b/tests/unit/command/test_data_ls.py
@@ -0,0 +1,96 @@
+import pytest
+
+from dvc.cli import parse_args
+from dvc.commands.data import CmdDataLs
+
+
+def test_cli(mocker):
+    mocker.patch("dvc.repo.Repo")
+    ls = mocker.patch("dvc.repo.data.ls", return_value={})
+    show_table_spy = mocker.spy(CmdDataLs, "_show_table")
+
+    cli_args = parse_args(
+        [
+            "data",
+            "ls",
+            "--labels",
+            "label1,label2",
+            "--labels",
+            "label3",
+            "--type",
+            "type1,type2",
+            "--type",
+            "type3",
+            "target",
+            "--recursive",
+            "--md",
+        ]
+    )
+
+    assert cli_args.func == CmdDataLs
+    cmd = cli_args.func(cli_args)
+    assert cmd.run() == 0
+
+    ls.assert_called_once_with(
+        cmd.repo,
+        targets=["target"],
+        recursive=True,
+    )
+    show_table_spy.assert_called_once_with(
+        {},
+        filter_labels={"label1", "label2", "label3"},
+        filter_types={"type1", "type2", "type3"},
+        markdown=True,
+    )
+
+
+EXAMPLE_DATA = [
+    {
+        "path": "model.pkl",
+        "desc": "desc",
+        "type": "model",
+        "labels": ["get-started", "dataset-registry"],
+        "meta": {"key": "value", "key1": "value1"},
+    },
+    {
+        "path": "model.pkl",
+        "desc": "desc",
+        "type": "model",
+        "labels": ["get-started", "example"],
+    },
+    {"path": "unlabeled.txt"},
+]
+
+
+@pytest.mark.parametrize(
+    "markdown, expected_output",
+    [
+        (
+            True,
+            """\
+| Path      | Type   | Labels                       | Description   |
+|-----------|--------|------------------------------|---------------|
+| model.pkl | model  | get-started,dataset-registry | desc          |
+| model.pkl | model  | get-started,example          | desc          |\n""",
+        ),
+        (
+            False,
+            """\
+ Path       Type   Labels                        Description
+ model.pkl  model  get-started,dataset-registry  desc
+ model.pkl  model  get-started,example           desc""",
+        ),
+    ],
+    ids=["markdown", "default"],
+)
+def test_ls(capsys, markdown, expected_output):
+    CmdDataLs._show_table(
+        EXAMPLE_DATA,
+        filter_types={"model"},
+        filter_labels={"get-started", "dataset-registry"},
+        markdown=markdown,
+    )
+    out, err = capsys.readouterr()
+    assert not err
+    out = "\n".join(line.rstrip() for line in out.splitlines())
+    assert out == expected_output