Repository: iterative/dvc

Issue:
plots: code templates not found
# Bug Report

Since 2.8.2 my code base templates are not located as they used to be via `dvc plots show/diff -t <template_name>`.

As far as I can tell this is due to the path root being prefixed to the template name before matching

https://github.com/iterative/dvc/blob/e2d705fd76771c184f85a477f8d73aea40a96cab/dvc/repo/plots/template.py#L123-L129

At face value the fix may just be 

```python
templates = [
    file
    for root, _, files in os.walk(self.templates_dir)
    for file in files
]
```

Discussion:

Patch:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4abb83318..26435a816 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@ repos:
       - id: black
         language_version: python3
     repo: https://github.com/ambv/black
-    rev: 21.9b0
+    rev: 21.10b0
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
     hooks:
diff --git a/dvc/command/plots.py b/dvc/command/plots.py
index 36241a61d..28611fb8d 100644
--- a/dvc/command/plots.py
+++ b/dvc/command/plots.py
@@ -68,8 +68,12 @@ class CmdPlots(CmdBase):
 
             if self.args.open:
                 import webbrowser
+                from platform import uname
 
-                opened = webbrowser.open(index_path)
+                if "Microsoft" in uname().release:
+                    url = Path(rel) / "index.html"
+
+                opened = webbrowser.open(url)
                 if not opened:
                     ui.error_write(
                         "Failed to open. Please try opening it manually."
diff --git a/dvc/compare.py b/dvc/compare.py
index 7808359b2..9384d5f1f 100644
--- a/dvc/compare.py
+++ b/dvc/compare.py
@@ -12,6 +12,7 @@ from typing import (
     Mapping,
     MutableSequence,
     Sequence,
+    Set,
     Tuple,
     Union,
     overload,
@@ -181,6 +182,35 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
             {k: self._columns[k][i] for k in keys} for i in range(len(self))
         ]
 
+    def dropna(self, axis: str = "rows"):
+        if axis not in ["rows", "cols"]:
+            raise ValueError(
+                f"Invalid 'axis' value {axis}."
+                "Choose one of ['rows', 'cols']"
+            )
+
+        to_drop: Set = set()
+        for n_row, row in enumerate(self):
+            for n_col, col in enumerate(row):
+                if col == self._fill_value:
+                    if axis == "rows":
+                        to_drop.add(n_row)
+                        break
+                    else:
+                        to_drop.add(self.keys()[n_col])
+
+        if axis == "rows":
+            for name in self.keys():
+                self._columns[name] = Column(
+                    [
+                        x
+                        for n, x in enumerate(self._columns[name])
+                        if n not in to_drop
+                    ]
+                )
+        else:
+            self.drop(*to_drop)
+
 
 def _normalize_float(val: float, precision: int):
     return f"{val:.{precision}g}"
diff --git a/dvc/output.py b/dvc/output.py
index d12ea209d..6424142d5 100644
--- a/dvc/output.py
+++ b/dvc/output.py
@@ -555,10 +555,6 @@ class Output:
 
         assert not self.IS_DEPENDENCY
 
-        if not self.changed():
-            logger.debug("Output '%s' didn't change. Skipping saving.", self)
-            return
-
         _, self.meta, self.obj = ostage(
             self.odb,
             self.path_info,
diff --git a/dvc/repo/experiments/init.py b/dvc/repo/experiments/init.py
index 69f6e13a4..8e488bca5 100644
--- a/dvc/repo/experiments/init.py
+++ b/dvc/repo/experiments/init.py
@@ -4,7 +4,7 @@ from contextlib import contextmanager
 from typing import TYPE_CHECKING, Dict, Iterable, cast
 
 from funcy import compact, lremove, post_processing
-from rich.prompt import Prompt
+from rich.prompt import Confirm, Prompt
 from rich.rule import Rule
 from rich.syntax import Syntax
 
@@ -33,7 +33,37 @@ PROMPTS = {
 }
 
 
-class RequiredPrompt(Prompt):
+class RichInputMixin:
+    """Prevents exc message from printing in the same line on Ctrl + D/C."""
+
+    @classmethod
+    def get_input(cls, *args, **kwargs) -> str:
+        try:
+            return super().get_input(*args, **kwargs)  # type: ignore[misc]
+        except (KeyboardInterrupt, EOFError):
+            ui.error_write()
+            raise
+
+
+class ConfirmationPrompt(RichInputMixin, Confirm):
+    def make_prompt(self, default):
+        prompt = self.prompt.copy()
+        prompt.end = ""
+
+        prompt.append(" [")
+        yes, no = self.choices
+        for idx, (val, choice) in enumerate(((True, yes), (False, no))):
+            if idx:
+                prompt.append("/")
+            if val == default:
+                prompt.append(choice.upper(), "green")
+            else:
+                prompt.append(choice)
+        prompt.append("] ")
+        return prompt
+
+
+class RequiredPrompt(RichInputMixin, Prompt):
     def process_response(self, value: str):
         from rich.prompt import InvalidResponse
 
@@ -44,14 +74,6 @@ class RequiredPrompt(Prompt):
             )
         return ret
 
-    @classmethod
-    def get_input(cls, *args, **kwargs) -> str:
-        try:
-            return super().get_input(*args, **kwargs)
-        except (KeyboardInterrupt, EOFError):
-            ui.error_write()
-            raise
-
     def render_default(self, default):
         from rich.text import Text
 
@@ -131,8 +153,10 @@ def init_interactive(
         provided.keys(), ["live"] if live else ["metrics", "plots"]
     )
 
+    ret: Dict[str, str] = {}
+
     if not (primary or secondary):
-        return {}
+        return ret
 
     message = ui.rich_text.assemble(
         "This command will guide you to set up a ",
@@ -162,12 +186,9 @@ def init_interactive(
         ui.error_write(tree, styled=True)
         ui.error_write()
 
-    return compact(
-        {
-            **_prompts(primary, defaults),
-            **_prompts(secondary, defaults),
-        }
-    )
+    ret.update(_prompts(primary, defaults))
+    ret.update(_prompts(secondary, defaults))
+    return compact(ret)
 
 
 def _check_stage_exists(
@@ -224,13 +245,13 @@ def init(
     assert "cmd" in context
 
     params_kv = []
-    if context.get("params"):
+    params = context.get("params")
+    if params:
         from dvc.utils.serialize import LOADERS
 
-        path = context["params"]
-        assert isinstance(path, str)
-        _, ext = os.path.splitext(path)
-        params_kv = [{path: list(LOADERS[ext](path))}]
+        assert isinstance(params, str)
+        _, ext = os.path.splitext(params)
+        params_kv = [{params: list(LOADERS[ext](params))}]
 
     checkpoint_out = bool(context.get("live"))
     models = context.get("models")
@@ -252,12 +273,16 @@ def init(
         syn = Syntax(_yaml, "yaml", theme="ansi_dark")
         ui.error_write(syn, styled=True)
 
-    if not interactive or ui.confirm(
-        "Do you want to add the above contents to dvc.yaml?"
+    if not interactive or ConfirmationPrompt.ask(
+        "Do you want to add the above contents to dvc.yaml?",
+        console=ui.error_console,
+        default=True,
     ):
-        with _disable_logging():
+        scm = repo.scm
+        with _disable_logging(), scm.track_file_changes(autostage=True):
             stage.dump(update_lock=False)
-        stage.ignore_outs()
+            stage.ignore_outs()
+            scm.track_file(params)
     else:
         raise DvcException("Aborting ...")
     return stage
diff --git a/dvc/repo/live.py b/dvc/repo/live.py
index f53007e89..9a7521395 100644
--- a/dvc/repo/live.py
+++ b/dvc/repo/live.py
@@ -1,5 +1,4 @@
 import logging
-import os
 from typing import TYPE_CHECKING, List, Optional
 
 from dvc.render.utils import render
@@ -19,10 +18,7 @@ def create_summary(out):
 
     html_path = out.path_info.fspath + "_dvc_plots"
 
-    index_path = render(
-        out.repo, plots, metrics=metrics, path=html_path, refresh_seconds=5
-    )
-    logger.info(f"\nfile://{os.path.abspath(index_path)}")
+    render(out.repo, plots, metrics=metrics, path=html_path, refresh_seconds=5)
 
 
 def summary_path_info(out: "Output") -> Optional["PathInfo"]:
diff --git a/dvc/repo/plots/template.py b/dvc/repo/plots/template.py
index 1e1384dd9..4e8cab56f 100644
--- a/dvc/repo/plots/template.py
+++ b/dvc/repo/plots/template.py
@@ -109,16 +109,16 @@ class PlotTemplates:
     @staticmethod
     def _find(templates, template_name):
         for template in templates:
-            if (
-                template_name == template
-                or template_name + ".json" == template
+            if template.endswith(template_name) or template.endswith(
+                template_name + ".json"
             ):
                 return template
         return None
 
     def _find_in_project(self, name: str) -> Optional["StrPath"]:
-        if os.path.exists(name):
-            return name
+        full_path = os.path.abspath(name)
+        if os.path.exists(full_path):
+            return full_path
 
         if os.path.exists(self.templates_dir):
             templates = [
diff --git a/dvc/scm/base.py b/dvc/scm/base.py
index 12f70f0e0..6a7950aa2 100644
--- a/dvc/scm/base.py
+++ b/dvc/scm/base.py
@@ -183,8 +183,10 @@ class Base:
         """
 
     @contextmanager
-    def track_file_changes(self, config=None):
-        autostage = get_in(config or {}, ["core", "autostage"])
+    def track_file_changes(self, config=None, autostage=False):
+        autostage = get_in(
+            config or {}, ["core", "autostage"], default=autostage
+        )
 
         try:
             yield
diff --git a/dvc/stage/monitor.py b/dvc/stage/monitor.py
index 9461c6022..03b05dab9 100644
--- a/dvc/stage/monitor.py
+++ b/dvc/stage/monitor.py
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Callable, List
 from dvc.repo.live import create_summary
 from dvc.stage.decorators import relock_repo
 from dvc.stage.exceptions import StageCmdFailedError
+from dvc.ui import ui
 
 if TYPE_CHECKING:
     from dvc.output import Output
@@ -86,11 +87,16 @@ class LiveTask(MonitorTask):
     error_cls = LiveKilledError
 
     def __init__(self, stage: "Stage", out: "Output", proc: subprocess.Popen):
+        self.output_path = os.path.join(
+            os.getcwd(), out.path_info.fspath + "_dvc_plots", "index.html"
+        )
+        ui.write(f"Live summary will be created at:\n{self.output_path}")
         super().__init__(stage, functools.partial(create_summary, out), proc)
 
     def after_run(self):
         # make sure summary is prepared for all the data
         self.execute()
+        ui.write(f"Live summary has been created at:\n{self.output_path}")
 
 
 class Monitor:
diff --git a/scripts/build-requirements.txt b/scripts/build-requirements.txt
index 93c47a281..d86645272 100644
--- a/scripts/build-requirements.txt
+++ b/scripts/build-requirements.txt
@@ -1,2 +1,2 @@
-PyInstaller==4.5.1
+PyInstaller==4.6
 numpy==1.19.3
diff --git a/setup.cfg b/setup.cfg
index 544cdc68f..c731000f4 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -102,7 +102,7 @@ azure =
 gdrive = pydrive2[fsspec]>=1.9.4
 gs = gcsfs==2021.10.1
 hdfs =
-    fsspec[arrow]; python_version < '3.10'
+    fsspec[arrow]
 oss = ossfs==2021.8.0
 s3 =
     s3fs==2021.10.1
@@ -128,7 +128,7 @@ tests =
     rangehttpserver==1.2.0
     mock-ssh-server==0.9.1
     wget==3.2
-    filelock==3.3.1
+    filelock==3.3.2
     wsgidav==3.1.1
     crc32c==2.2.post0
     xmltodict==0.12.0


Test patch:
diff --git a/tests/remotes/hdfs.py b/tests/remotes/hdfs.py
index e01b2adcd..4a44961d5 100644
--- a/tests/remotes/hdfs.py
+++ b/tests/remotes/hdfs.py
@@ -2,7 +2,6 @@ import locale
 import os
 import platform
 import subprocess
-import sys
 import uuid
 from contextlib import contextmanager
 from pathlib import Path
@@ -74,10 +73,7 @@ class HDFS(Base, URLInfo):  # pylint: disable=abstract-method
 @pytest.fixture(scope="session")
 def hadoop(test_config):
     test_config.requires("real_hdfs")
-
-    if sys.version_info >= (3, 10):
-        pytest.skip("pyarrow is not available for 3.10 yet.")
-
+    pytest.importorskip("pyarrow.fs")
     if platform.system() != "Linux":
         pytest.skip("only supported on Linux")
 
@@ -280,9 +276,7 @@ def hdfs(test_config, mocker):
     # "The pyarrow installation is not built with support for
     # 'HadoopFileSystem'"
     test_config.requires("hdfs")
-
-    if sys.version_info >= (3, 10):
-        pytest.skip("pyarrow is not available for 3.10 yet.")
+    pytest.importorskip("pyarrow.fs")
 
     mocker.patch("pyarrow.fs._not_imported", [])
     mocker.patch(
diff --git a/tests/unit/command/test_plots.py b/tests/unit/command/test_plots.py
index 0387f90f7..a9258a35f 100644
--- a/tests/unit/command/test_plots.py
+++ b/tests/unit/command/test_plots.py
@@ -1,5 +1,6 @@
 import os
 import posixpath
+from pathlib import Path
 
 import pytest
 
@@ -149,12 +150,31 @@ def test_plots_diff_open(tmp_dir, dvc, mocker, capsys, plots_data):
     mocker.patch("dvc.command.plots.render", return_value=index_path)
 
     assert cmd.run() == 0
-    mocked_open.assert_called_once_with(index_path)
+    mocked_open.assert_called_once_with(index_path.as_uri())
 
     out, _ = capsys.readouterr()
     assert index_path.as_uri() in out
 
 
+def test_plots_diff_open_WSL(tmp_dir, dvc, mocker, plots_data):
+    mocked_open = mocker.patch("webbrowser.open", return_value=True)
+    mocked_uname_result = mocker.MagicMock()
+    mocked_uname_result.release = "Microsoft"
+    mocker.patch("platform.uname", return_value=mocked_uname_result)
+
+    cli_args = parse_args(
+        ["plots", "diff", "--targets", "plots.csv", "--open"]
+    )
+    cmd = cli_args.func(cli_args)
+    mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data)
+
+    index_path = tmp_dir / "dvc_plots" / "index.html"
+    mocker.patch("dvc.command.plots.render", return_value=index_path)
+
+    assert cmd.run() == 0
+    mocked_open.assert_called_once_with(Path("dvc_plots") / "index.html")
+
+
 def test_plots_diff_open_failed(tmp_dir, dvc, mocker, capsys, plots_data):
     mocked_open = mocker.patch("webbrowser.open", return_value=False)
     cli_args = parse_args(
@@ -167,7 +187,7 @@ def test_plots_diff_open_failed(tmp_dir, dvc, mocker, capsys, plots_data):
 
     assert cmd.run() == 1
     expected_url = tmp_dir / "dvc_plots" / "index.html"
-    mocked_open.assert_called_once_with(expected_url)
+    mocked_open.assert_called_once_with(expected_url.as_uri())
 
     error_message = "Failed to open. Please try opening it manually."
 
diff --git a/tests/unit/render/test_vega.py b/tests/unit/render/test_vega.py
index 0b3d936c6..5ff80b55c 100644
--- a/tests/unit/render/test_vega.py
+++ b/tests/unit/render/test_vega.py
@@ -451,3 +451,29 @@ def test_find_vega(tmp_dir, dvc):
         first(plot_content["layer"])["encoding"]["x"]["field"] == INDEX_FIELD
     )
     assert first(plot_content["layer"])["encoding"]["y"]["field"] == "y"
+
+
+@pytest.mark.parametrize(
+    "template_path, target_name",
+    [
+        (os.path.join(".dvc", "plots", "template.json"), "template"),
+        (os.path.join(".dvc", "plots", "template.json"), "template.json"),
+        (
+            os.path.join(".dvc", "plots", "subdir", "template.json"),
+            os.path.join("subdir", "template.json"),
+        ),
+        (
+            os.path.join(".dvc", "plots", "subdir", "template.json"),
+            os.path.join("subdir", "template"),
+        ),
+        ("template.json", "template.json"),
+    ],
+)
+def test_should_resolve_template(tmp_dir, dvc, template_path, target_name):
+    os.makedirs(os.path.abspath(os.path.dirname(template_path)), exist_ok=True)
+    with open(template_path, "w", encoding="utf-8") as fd:
+        fd.write("template_content")
+
+    assert dvc.plots.templates._find_in_project(
+        target_name
+    ) == os.path.abspath(template_path)
diff --git a/tests/unit/test_tabular_data.py b/tests/unit/test_tabular_data.py
index 0b815eae7..9631c4362 100644
--- a/tests/unit/test_tabular_data.py
+++ b/tests/unit/test_tabular_data.py
@@ -177,3 +177,36 @@ def test_row_from_dict():
         ["", "", "value3", "value4", "", ""],
         ["", "", "value3", "", "value5", "value6"],
     ]
+
+
+@pytest.mark.parametrize(
+    "axis,expected",
+    [
+        (
+            "rows",
+            [
+                ["foo", "bar", "foobar"],
+            ],
+        ),
+        ("cols", [["foo"], ["foo"], ["foo"]]),
+    ],
+)
+def test_dropna(axis, expected):
+    td = TabularData(["col-1", "col-2", "col-3"])
+    td.extend([["foo"], ["foo", "bar"], ["foo", "bar", "foobar"]])
+    assert list(td) == [
+        ["foo", "", ""],
+        ["foo", "bar", ""],
+        ["foo", "bar", "foobar"],
+    ]
+
+    td.dropna(axis)
+
+    assert list(td) == expected
+
+
+def test_dropna_invalid_axis():
+    td = TabularData(["col-1", "col-2", "col-3"])
+
+    with pytest.raises(ValueError, match="Invalid 'axis' value foo."):
+        td.dropna("foo")