Repository: iterative/dvc

Issue:
plots: integration tests for vscode
The `--json` option for `dvc plots` subcommands needs integration tests that would serve as a contract between dvc and vscode.

Current use cases: 
-  plots with and without some properties - make sure proper JSON is produced
- are no plots in the repo at all
-  verify that data types for JSON structure are as expected by vscode

cc @mattseddon 
Related : iterative/vscode-dvc#1521 iterative/vscode-dvc#1520


Discussion:

Resolving patch:
diff --git a/dvc/commands/plots.py b/dvc/commands/plots.py
--- a/dvc/commands/plots.py
+++ b/dvc/commands/plots.py
@@ -70,8 +70,13 @@ def run(self):
                     "visualization file will not be created."
                 )
 
+            out: str = self.args.out or "dvc_plots"
+
+            renderers_out = (
+                out if self.args.json else os.path.join(out, "static")
+            )
             renderers = match_renderers(
-                plots_data=plots_data, out=self.args.out
+                plots_data=plots_data, out=renderers_out
             )
 
             if self.args.show_vega:
@@ -95,8 +100,7 @@ def run(self):
                         self.repo.dvc_dir, html_template_path
                     )
 
-            rel: str = self.args.out or "dvc_plots"
-            output_file: Path = (Path.cwd() / rel).resolve() / "index.html"
+            output_file: Path = (Path.cwd() / out).resolve() / "index.html"
 
             render_html(
                 renderers=renderers,
diff --git a/dvc/render/convert.py b/dvc/render/convert.py
--- a/dvc/render/convert.py
+++ b/dvc/render/convert.py
@@ -1,4 +1,5 @@
 import json
+import os
 from collections import defaultdict
 from typing import Dict, List, Union
 
@@ -23,6 +24,7 @@ def _get_converter(
 def to_datapoints(renderer_class, data: Dict, props: Dict):
     converter = _get_converter(renderer_class, props)
     datapoints: List[Dict] = []
+    final_props: Dict = {}
     for revision, rev_data in data.items():
         for filename, file_data in rev_data.get("data", {}).items():
             if "data" in file_data:
@@ -42,8 +44,10 @@ def _group_by_rev(datapoints):
 
 
 def to_json(renderer, split: bool = False) -> List[Dict]:
+    from copy import deepcopy
+
     if renderer.TYPE == "vega":
-        grouped = _group_by_rev(renderer.datapoints)
+        grouped = _group_by_rev(deepcopy(renderer.datapoints))
         if split:
             content = renderer.get_filled_template(skip_anchors=["data"])
         else:
@@ -61,7 +65,7 @@ def to_json(renderer, split: bool = False) -> List[Dict]:
             {
                 TYPE_KEY: renderer.TYPE,
                 REVISIONS_KEY: [datapoint.get(REVISION_FIELD)],
-                "url": datapoint.get(SRC_FIELD),
+                "url": os.path.abspath(datapoint.get(SRC_FIELD)),
             }
             for datapoint in renderer.datapoints
         ]
diff --git a/dvc/render/image_converter.py b/dvc/render/image_converter.py
--- a/dvc/render/image_converter.py
+++ b/dvc/render/image_converter.py
@@ -45,9 +45,7 @@ def convert(
         if path:
             if not os.path.isdir(path):
                 os.makedirs(path, exist_ok=True)
-            src = self._write_image(
-                os.path.abspath(path), revision, filename, data
-            )
+            src = self._write_image(path, revision, filename, data)
         else:
             src = self._encode_image(data)
         datapoint = {


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -215,3 +215,38 @@ def isolate(tmp_path_factory, monkeypatch) -> None:
     import pygit2
 
     pygit2.settings.search_path[pygit2.GIT_CONFIG_LEVEL_GLOBAL] = str(home_dir)
+
+
+@pytest.fixture
+def run_copy_metrics(tmp_dir, run_copy):
+    def run(
+        file1,
+        file2,
+        commit=None,
+        tag=None,
+        single_stage=True,
+        name=None,
+        **kwargs,
+    ):
+        if name:
+            single_stage = False
+
+        stage = tmp_dir.dvc.run(
+            cmd=f"python copy.py {file1} {file2}",
+            deps=[file1],
+            single_stage=single_stage,
+            name=name,
+            **kwargs,
+        )
+
+        if hasattr(tmp_dir.dvc, "scm"):
+            files = [stage.path]
+            files += [out.fs_path for out in stage.outs if not out.use_cache]
+            tmp_dir.dvc.scm.add(files)
+            if commit:
+                tmp_dir.dvc.scm.commit(commit)
+            if tag:
+                tmp_dir.dvc.scm.tag(tag)
+        return stage
+
+    return run
diff --git a/tests/func/conftest.py b/tests/func/conftest.py
deleted file mode 100644
--- a/tests/func/conftest.py
+++ /dev/null
@@ -1,36 +0,0 @@
-import pytest
-
-
-@pytest.fixture
-def run_copy_metrics(tmp_dir, run_copy):
-    def run(
-        file1,
-        file2,
-        commit=None,
-        tag=None,
-        single_stage=True,
-        name=None,
-        **kwargs,
-    ):
-        if name:
-            single_stage = False
-
-        stage = tmp_dir.dvc.run(
-            cmd=f"python copy.py {file1} {file2}",
-            deps=[file1],
-            single_stage=single_stage,
-            name=name,
-            **kwargs,
-        )
-
-        if hasattr(tmp_dir.dvc, "scm"):
-            files = [stage.path]
-            files += [out.fs_path for out in stage.outs if not out.use_cache]
-            tmp_dir.dvc.scm.add(files)
-            if commit:
-                tmp_dir.dvc.scm.commit(commit)
-            if tag:
-                tmp_dir.dvc.scm.tag(tag)
-        return stage
-
-    return run
diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
new file mode 100644
diff --git a/tests/integration/plots/__init__.py b/tests/integration/plots/__init__.py
new file mode 100644
diff --git a/tests/integration/plots/conftest.py b/tests/integration/plots/conftest.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/plots/conftest.py
@@ -0,0 +1,81 @@
+import pytest
+
+
+@pytest.fixture
+def repo_with_plots(tmp_dir, scm, dvc, run_copy_metrics):
+    def make():
+        linear_v1 = [
+            {"x": 1, "y": 0.1},
+            {"x": 2, "y": 0.2},
+            {"x": 3, "y": 0.3},
+        ]
+
+        confusion_v1 = [
+            {"actual": 0, "predicted": 1},
+            {"actual": 0, "predicted": 1},
+            {"actual": 1, "predicted": 0},
+            {"actual": 1, "predicted": 0},
+        ]
+
+        image_v1 = b"content"
+
+        (tmp_dir / "linear_src.json").dump_json(linear_v1)
+        (tmp_dir / "confusion_src.json").dump_json(confusion_v1)
+        (tmp_dir / "image_src.png").write_bytes(image_v1)
+
+        scm.add(["linear_src.json", "confusion_src.json", "image_src.png"])
+        scm.commit("add data sources")
+
+        run_copy_metrics(
+            "linear_src.json",
+            "linear.json",
+            name="linear",
+            plots=["linear.json"],
+            commit="linear",
+        )
+        run_copy_metrics(
+            "confusion_src.json",
+            "confusion.json",
+            name="confusion",
+            plots=["confusion.json"],
+            commit="confusion",
+        )
+        confusion_props = {
+            "title": "confusion matrix",
+            "x": "predicted",
+            "y": "actual",
+            "template": "confusion",
+        }
+        dvc.plots.modify("confusion.json", confusion_props)
+        run_copy_metrics(
+            "image_src.png",
+            "image.png",
+            name="image",
+            plots=["image.png"],
+            commit="image",
+        )
+
+        scm.add(["dvc.yaml", "dvc.lock"])
+        scm.commit("commit dvc files")
+        yield image_v1, linear_v1, confusion_v1, confusion_props
+        linear_v2 = [
+            {"x": 1, "y": 0.2},
+            {"x": 2, "y": 0.3},
+            {"x": 3, "y": 0.4},
+        ]
+        confusion_v2 = [
+            {"actual": 0, "predicted": 0},
+            {"actual": 0, "predicted": 0},
+            {"actual": 1, "predicted": 1},
+            {"actual": 1, "predicted": 1},
+        ]
+        image_v2 = b"content2"
+
+        (tmp_dir / "linear_src.json").dump_json(linear_v2)
+        (tmp_dir / "confusion_src.json").dump_json(confusion_v2)
+        (tmp_dir / "image_src.png").write_bytes(image_v2)
+
+        dvc.reproduce()
+        yield image_v2, linear_v2, confusion_v2, confusion_props
+
+    return make
diff --git a/tests/integration/plots/test_json.py b/tests/integration/plots/test_json.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/plots/test_json.py
@@ -0,0 +1,246 @@
+import json
+import os
+from copy import deepcopy
+from typing import Dict, List
+from urllib.parse import urlparse
+from urllib.request import url2pathname
+
+import dpath.util
+import pytest
+from funcy import first
+
+from dvc.cli import main
+
+JSON_OUT = "vis_data"
+
+
+def call(capsys, subcommand="show"):
+    capsys.readouterr()
+    assert (
+        main(["plots", subcommand, "--json", "-o", JSON_OUT, "--split"]) == 0
+    )
+    split_json_out, _ = capsys.readouterr()
+
+    split_json_result = json.loads(split_json_out)
+
+    capsys.readouterr()
+    assert main(["plots", subcommand, "--json", "-o", JSON_OUT]) == 0
+    json_out, _ = capsys.readouterr()
+
+    json_result = json.loads(json_out)
+
+    assert main(["plots", subcommand]) == 0
+    html_path_out, _ = capsys.readouterr()
+
+    parsed = urlparse(html_path_out.strip())
+    abspath = url2pathname(parsed.path)
+    return abspath, json_result, split_json_result
+
+
+def filter_fields(datapoints: List[Dict], fields: List[str]):
+
+    tmp = deepcopy(datapoints)
+    for datapoint in tmp:
+        keys = set(datapoint.keys())
+        for key in keys:
+            if key not in fields:
+                datapoint.pop(key)
+    return tmp
+
+
+def verify_image(tmp_dir, version, filename, content, html_path, json_result):
+    assert os.path.exists(html_path)
+    with open(html_path, "r", encoding="utf-8") as fd:
+        html_content = fd.read()
+
+    image_data = {}
+    for datapoint in json_result[filename]:
+        if datapoint["revisions"] == [version]:
+            image_data = datapoint
+            break
+    assert image_data, f"{version} data for {filename} was not found"
+    assert image_data["type"] == "image"
+    assert image_data["url"] == str(
+        tmp_dir / JSON_OUT / f"{version}_{filename}"
+    )
+    assert (
+        tmp_dir / JSON_OUT / f"{version}_{filename}"
+    ).read_bytes() == content
+
+    assert (
+        os.path.join("dvc_plots", "static", f"{version}_{filename}")
+        in html_content
+    )
+
+    # there should be no absolute paths in produced HTML
+    assert str(tmp_dir) not in html_content
+    assert (
+        tmp_dir / "dvc_plots" / "static" / f"{version}_{filename}"
+    ).read_bytes() == content
+
+
+def _remove_blanks(text: str):
+    return text.replace("\t", "").replace("\n", "").replace(" ", "")
+
+
+def verify_vega(
+    versions,
+    filename,
+    fields,
+    expected_data,
+    html_path,
+    json_result,
+    split_json_result,
+):
+
+    assert os.path.exists(html_path)
+    with open(html_path, "r", encoding="utf-8") as fd:
+        html_content = fd.read()
+    assert _remove_blanks(
+        json.dumps(dpath.util.get(json_result, [filename, 0, "content"]))
+    ) in _remove_blanks(html_content)
+
+    if isinstance(versions, str):
+        versions = [versions]
+
+    for j in [json_result, split_json_result]:
+        assert len(j[filename]) == 1
+        assert dpath.util.get(j, [filename, 0, "type"]) == "vega"
+        assert dpath.util.get(j, [filename, 0, "revisions"]) == versions
+
+    assert dpath.util.get(
+        json_result, [filename, 0, "datapoints"]
+    ) == dpath.util.get(split_json_result, [filename, 0, "datapoints"])
+    assert set(versions) == set(
+        dpath.util.get(json_result, [filename, 0, "datapoints"]).keys()
+    )
+
+    assert (
+        filter_fields(
+            dpath.util.get(
+                json_result, [filename, 0, "content", "data", "values"]
+            ),
+            fields,
+        )
+        == expected_data
+    )
+
+    assert (
+        dpath.util.get(
+            split_json_result, [filename, 0, "content", "data", "values"]
+        )
+        == "<DVC_METRIC_DATA>"
+    )
+
+    # if data is not considered, json and split json should be identical
+    json_content = deepcopy(
+        dpath.util.get(json_result, [filename, 0, "content"])
+    )
+    split_json_content = deepcopy(
+        dpath.util.get(split_json_result, [filename, 0, "content"])
+    )
+    dpath.util.set(json_content, ["data"], {})
+    dpath.util.set(split_json_content, ["data"], {})
+    assert json_content == split_json_content
+
+
+def verify_vega_props(filename, json_result, title, x, y, **kwargs):
+    data = json_result[filename]
+    assert len(data) == 1
+    data = first(data)
+
+    assert dpath.util.get(data, ["content", "title"]) == title
+
+    assert (
+        dpath.util.get(data, ["content", "spec", "encoding", "x", "field"])
+        == x
+    )
+    assert (
+        dpath.util.get(data, ["content", "spec", "encoding", "y", "field"])
+        == y
+    )
+
+
+@pytest.mark.vscode
+def test_no_plots(tmp_dir, scm, dvc, capsys):
+    html_path, json_result, split_json_result = call(capsys)
+    assert os.path.exists(html_path)
+    assert json_result == {}
+    assert split_json_result == {}
+
+
+@pytest.mark.vscode
+def test_repo_with_plots(
+    tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_plots
+):
+    repo_state = repo_with_plots()
+
+    image_v1, linear_v1, confusion_v1, confusion_props = next(repo_state)
+
+    html_path, json_result, split_json_result = call(capsys)
+
+    verify_image(
+        tmp_dir, "workspace", "image.png", image_v1, html_path, json_result
+    )
+
+    verify_vega(
+        "workspace",
+        "linear.json",
+        ["x", "y"],
+        linear_v1,
+        html_path,
+        json_result,
+        split_json_result,
+    )
+
+    verify_vega(
+        "workspace",
+        "confusion.json",
+        ["actual", "predicted"],
+        confusion_v1,
+        html_path,
+        json_result,
+        split_json_result,
+    )
+    verify_vega_props("confusion.json", json_result, **confusion_props)
+
+    image_v2, linear_v2, confusion_v2, confusion_props = next(repo_state)
+
+    html_path, json_result, split_json_result = call(capsys, subcommand="diff")
+
+    verify_image(
+        tmp_dir, "workspace", "image.png", image_v2, html_path, json_result
+    )
+    verify_image(
+        tmp_dir, "HEAD", "image.png", image_v1, html_path, json_result
+    )
+
+    verify_vega(
+        ["HEAD", "workspace"],
+        "linear.json",
+        ["x", "y"],
+        linear_v2 + linear_v1,
+        html_path,
+        json_result,
+        split_json_result,
+    )
+
+    verify_vega(
+        ["HEAD", "workspace"],
+        "confusion.json",
+        ["actual", "predicted"],
+        confusion_v2 + confusion_v1,
+        html_path,
+        json_result,
+        split_json_result,
+    )
+    verify_vega_props("confusion.json", json_result, **confusion_props)
+    from dvc.utils.fs import remove
+
+    # even if there is no data, call should be successful
+    remove(tmp_dir / ".dvc" / "cache")
+    remove("linear.json")
+    remove("confusion.json")
+    remove("image.png")
+    call(capsys)
+    call(capsys, subcommand="diff")
diff --git a/tests/integration/plots/test_repo_plots_api.py b/tests/integration/plots/test_repo_plots_api.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/plots/test_repo_plots_api.py
@@ -0,0 +1,118 @@
+import dpath.util
+import pytest
+
+
+@pytest.mark.studio
+def test_api(tmp_dir, dvc, repo_with_plots):
+    repo_state = repo_with_plots()
+    image_v1, linear_v1, confusion_v1, confusion_params = next(repo_state)
+
+    workspace_data = next(dvc.plots.collect())
+
+    assert (
+        dpath.util.get(
+            workspace_data, ["workspace", "data", "image.png", "props"]
+        )
+        == {}
+    )
+    image_source = dpath.util.get(
+        workspace_data, ["workspace", "data", "image.png", "data_source"]
+    )
+    assert callable(image_source)
+    assert image_source() == {"data": image_v1}
+
+    assert (
+        dpath.util.get(
+            workspace_data, ["workspace", "data", "linear.json", "props"]
+        )
+        == {}
+    )
+    linear_source = dpath.util.get(
+        workspace_data, ["workspace", "data", "linear.json", "data_source"]
+    )
+    assert callable(linear_source)
+    assert linear_source() == {"data": linear_v1}
+
+    assert (
+        dpath.util.get(
+            workspace_data, ["workspace", "data", "confusion.json", "props"]
+        )
+        == confusion_params
+    )
+    confusion_source = dpath.util.get(
+        workspace_data, ["workspace", "data", "confusion.json", "data_source"]
+    )
+    assert callable(confusion_source)
+    assert confusion_source() == {"data": confusion_v1}
+
+    image_v2, linear_v2, confusion_v2, _ = next(repo_state)
+    data_generator = dvc.plots.collect(revs=["workspace", "HEAD"])
+
+    workspace_data = next(data_generator)
+
+    assert (
+        dpath.util.get(
+            workspace_data, ["workspace", "data", "image.png", "props"]
+        )
+        == {}
+    )
+    image_source = dpath.util.get(
+        workspace_data, ["workspace", "data", "image.png", "data_source"]
+    )
+    assert callable(image_source)
+    assert image_source() == {"data": image_v2}
+
+    assert (
+        dpath.util.get(
+            workspace_data, ["workspace", "data", "linear.json", "props"]
+        )
+        == {}
+    )
+    linear_source = dpath.util.get(
+        workspace_data, ["workspace", "data", "linear.json", "data_source"]
+    )
+    assert callable(linear_source)
+    assert linear_source() == {"data": linear_v2}
+
+    assert (
+        dpath.util.get(
+            workspace_data, ["workspace", "data", "confusion.json", "props"]
+        )
+        == confusion_params
+    )
+    confusion_source = dpath.util.get(
+        workspace_data, ["workspace", "data", "confusion.json", "data_source"]
+    )
+    assert callable(confusion_source)
+    assert confusion_source() == {"data": confusion_v2}
+
+    head_data = next(data_generator)
+
+    assert (
+        dpath.util.get(head_data, ["HEAD", "data", "image.png", "props"]) == {}
+    )
+    image_source = dpath.util.get(
+        head_data, ["HEAD", "data", "image.png", "data_source"]
+    )
+    assert callable(image_source)
+    assert image_source() == {"data": image_v1}
+
+    assert (
+        dpath.util.get(head_data, ["HEAD", "data", "linear.json", "props"])
+        == {}
+    )
+    linear_source = dpath.util.get(
+        head_data, ["HEAD", "data", "linear.json", "data_source"]
+    )
+    assert callable(linear_source)
+    assert linear_source() == {"data": linear_v1}
+
+    assert (
+        dpath.util.get(head_data, ["HEAD", "data", "confusion.json", "props"])
+        == confusion_params
+    )
+    confusion_source = dpath.util.get(
+        head_data, ["HEAD", "data", "confusion.json", "data_source"]
+    )
+    assert callable(confusion_source)
+    assert confusion_source() == {"data": confusion_v1}
diff --git a/tests/unit/render/test_convert.py b/tests/unit/render/test_convert.py
--- a/tests/unit/render/test_convert.py
+++ b/tests/unit/render/test_convert.py
@@ -205,6 +205,8 @@ def test_to_json_vega_split(mocker):
 
 
 def test_to_json_image(mocker):
+    import os
+
     image_renderer = mocker.MagicMock()
     image_renderer.TYPE = "image"
     image_renderer.datapoints = [
@@ -213,7 +215,7 @@ def test_to_json_image(mocker):
     ]
     result = to_json(image_renderer)
     assert result[0] == {
-        "url": image_renderer.datapoints[0].get(SRC_FIELD),
+        "url": os.path.abspath(image_renderer.datapoints[0].get(SRC_FIELD)),
         REVISIONS_KEY: [image_renderer.datapoints[0].get(REVISION_FIELD)],
         TYPE_KEY: image_renderer.TYPE,
     }