Repository: iterative/dvc

Issue:
"dvc metrics show -a" doesn't work if a metric file is missing
> Hi everyone,

I have three branches. Every branch is one experiment. One experiment should create a metric file as output. Two experiments are already finished, the third experiment is still running. So I have two experiments (= branches) with a metric file and one without.

Now I want to use "dvc metrics show -a" to get the result from the two already finished experiments. But I got an error that the branch with the running experiment does not have a metric file.

I am working with the pip installed DVC (version 0.35.7). The system is a ubuntu 18.04 and use anaconda.

I got help from ruslan in the forum: https://discordapp.com/channels/485586884165107732/563406153334128681/570262271847039028

thanks for the help!

> cmd: python code/train.py 
> deps:
> - path: data/toy_3dpatients_on_mrt.npy
> - path: data/toy_3dpatients_on_mrt_settings.npy
> - path: data/toy_3dpatients_on_mrt_test.npy
> outs:
> - cache: false
>   metric: true
>   path: metrics/test_local_False.json
> - cache: true
>   metric: false
>   path: tensorboards/test_local_False_test
> - cache: true
>   metric: false
>   path: tensorboards/test_local_False_train
> - cache: true
>   metric: false
>   path: tensorflow_models/model_weights_test_local_False.h5

And here is the output from the command `dvc metrics show -v -a`
> (tf-gpu) j@t:~/Documents/gitlab/0005_3d_siamesenet$ dvc metrics show -v -a
> ERROR: unexpected error - [Errno 2] No such file in branch 'test_local': 'metrics/test_local_False.json'
> ------------------------------------------------------------
> Traceback (most recent call last):
>   File "/home/j/anaconda3/envs/tf-gpu/lib/python3.6/site-packages/dvc/main.py", line 37, in main
>     ret = cmd.run_cmd()
>   File "/home/j/anaconda3/envs/tf-gpu/lib/python3.6/site-packages/dvc/command/base.py", line 60, in run_cmd
>     return self.run()
>   File "/home/j/anaconda3/envs/tf-gpu/lib/python3.6/site-packages/dvc/command/metrics.py", line 47, in run
>     recursive=self.args.recursive,
>   File "/home/j/anaconda3/envs/tf-gpu/lib/python3.6/site-packages/dvc/repo/metrics/init.py", line 18, in show
>     return show(self.repo, *args, **kwargs)
>   File "/home/j/anaconda3/envs/tf-gpu/lib/python3.6/site-packages/dvc/repo/metrics/show.py", line 263, in show
>     metrics = _read_metrics(self, entries, branch)
>   File "/home/j/anaconda3/envs/tf-gpu/lib/python3.6/site-packages/dvc/repo/metrics/show.py", line 237, in _read_metrics
>     fd = self.tree.open(out.path)
>   File "/home/j/anaconda3/envs/tf-gpu/lib/python3.6/site-packages/dvc/scm/git/tree.py", line 34, in open
>     raise IOError(errno.ENOENT, msg, relpath)
> FileNotFoundError: [Errno 2] No such file in branch 'test_local': 'metrics/test_local_False.json'



Discussion:

Resolving patch:
diff --git a/dvc/repo/metrics/show.py b/dvc/repo/metrics/show.py
--- a/dvc/repo/metrics/show.py
+++ b/dvc/repo/metrics/show.py
@@ -1,14 +1,19 @@
 from __future__ import unicode_literals
 
+import errno
 import os
 import csv
 import json
 import logging
+
 from jsonpath_ng.ext import parse
 
 from dvc.exceptions import OutputNotFoundError, BadMetricError, NoMetricsError
 from dvc.utils.compat import builtin_str, open, StringIO, csv_reader
 
+NO_METRICS_FILE_AT_REFERENCE_WARNING = (
+    "Metrics file '{}' does not exist at reference: '{}'"
+)
 
 logger = logging.getLogger(__name__)
 
@@ -147,7 +152,7 @@ def _read_metric(fd, typ=None, xpath=None, rel_path=None, branch=None):
         return None
 
 
-def _collect_metrics(self, path, recursive, typ, xpath, branch):
+def _collect_metrics(repo, path, recursive, typ, xpath, branch):
     """Gather all the metric outputs.
 
     Args:
@@ -165,11 +170,11 @@ def _collect_metrics(self, path, recursive, typ, xpath, branch):
             - typ:
             - xpath:
     """
-    outs = [out for stage in self.stages() for out in stage.outs]
+    outs = [out for stage in repo.stages() for out in stage.outs]
 
     if path:
         try:
-            outs = self.find_outs_by_path(path, outs=outs, recursive=recursive)
+            outs = repo.find_outs_by_path(path, outs=outs, recursive=recursive)
         except OutputNotFoundError:
             logger.debug(
                 "stage file not for found for '{}' in branch '{}'".format(
@@ -195,16 +200,7 @@ def _collect_metrics(self, path, recursive, typ, xpath, branch):
     return res
 
 
-def _read_metrics_filesystem(path, typ, xpath, rel_path, branch):
-    if not os.path.exists(path):
-        return None
-    with open(path, "r") as fd:
-        return _read_metric(
-            fd, typ=typ, xpath=xpath, rel_path=rel_path, branch=branch
-        )
-
-
-def _read_metrics(self, metrics, branch):
+def _read_metrics(repo, metrics, branch):
     """Read the content of each metric file and format it.
 
     Args:
@@ -226,15 +222,14 @@ def _read_metrics(self, metrics, branch):
         if not typ:
             typ = os.path.splitext(out.path.lower())[1].replace(".", "")
         if out.use_cache:
-            metric = _read_metrics_filesystem(
-                self.cache.local.get(out.checksum),
-                typ=typ,
-                xpath=xpath,
-                rel_path=out.rel_path,
-                branch=branch,
-            )
+            open_fun = open
+            path = repo.cache.local.get(out.checksum)
         else:
-            with self.tree.open(out.path) as fd:
+            open_fun = repo.tree.open
+            path = out.path
+        try:
+
+            with open_fun(path) as fd:
                 metric = _read_metric(
                     fd,
                     typ=typ,
@@ -242,6 +237,16 @@ def _read_metrics(self, metrics, branch):
                     rel_path=out.rel_path,
                     branch=branch,
                 )
+        except IOError as e:
+            if e.errno == errno.ENOENT:
+                logger.warning(
+                    NO_METRICS_FILE_AT_REFERENCE_WARNING.format(
+                        out.rel_path, branch
+                    )
+                )
+                metric = None
+            else:
+                raise
 
         if not metric:
             continue
@@ -252,7 +257,7 @@ def _read_metrics(self, metrics, branch):
 
 
 def show(
-    self,
+    repo,
     path=None,
     typ=None,
     xpath=None,
@@ -262,9 +267,9 @@ def show(
 ):
     res = {}
 
-    for branch in self.brancher(all_branches=all_branches, all_tags=all_tags):
-        entries = _collect_metrics(self, path, recursive, typ, xpath, branch)
-        metrics = _read_metrics(self, entries, branch)
+    for branch in repo.brancher(all_branches=all_branches, all_tags=all_tags):
+        entries = _collect_metrics(repo, path, recursive, typ, xpath, branch)
+        metrics = _read_metrics(repo, entries, branch)
         if metrics:
             res[branch] = metrics
 


Test patch:
diff --git a/tests/func/test_metrics.py b/tests/func/test_metrics.py
--- a/tests/func/test_metrics.py
+++ b/tests/func/test_metrics.py
@@ -6,6 +6,8 @@
 from dvc.repo import Repo as DvcRepo
 from dvc.main import main
 from dvc.exceptions import DvcException, BadMetricError, NoMetricsError
+from dvc.repo.metrics.show import NO_METRICS_FILE_AT_REFERENCE_WARNING
+from dvc.stage import Stage
 from tests.basic_env import TestDvc
 
 
@@ -775,3 +777,50 @@ def _do_show(self, file_name, xpath):
                 self.assertSequenceEqual(ret[branch][file_name], [branch])
             else:
                 self.assertSequenceEqual(ret[branch][file_name], branch)
+
+
+class TestShouldDisplayMetricsEvenIfMetricIsMissing(object):
+    BRANCH_MISSING_METRIC = "missing_metric_branch"
+    METRIC_FILE = "metric"
+    METRIC_FILE_STAGE = METRIC_FILE + Stage.STAGE_FILE_SUFFIX
+
+    def _write_metric(self):
+        with open(self.METRIC_FILE, "w+") as fd:
+            fd.write("0.5")
+            fd.flush()
+
+    def _commit_metric(self, dvc, branch):
+        dvc.scm.add([self.METRIC_FILE_STAGE])
+        dvc.scm.commit("{} commit".format(branch))
+
+    def setUp(self, dvc):
+        dvc.scm.branch(self.BRANCH_MISSING_METRIC)
+
+        self._write_metric()
+
+        ret = main(["run", "-m", self.METRIC_FILE])
+        assert 0 == ret
+
+        self._commit_metric(dvc, "master")
+
+    def test(self, dvc, caplog):
+        self.setUp(dvc)
+
+        dvc.scm.checkout(self.BRANCH_MISSING_METRIC)
+
+        self._write_metric()
+        ret = main(["run", "-M", self.METRIC_FILE])
+        assert 0 == ret
+
+        self._commit_metric(dvc, self.BRANCH_MISSING_METRIC)
+        os.remove(self.METRIC_FILE)
+
+        ret = main(["metrics", "show", "-a"])
+
+        assert (
+            NO_METRICS_FILE_AT_REFERENCE_WARNING.format(
+                self.METRIC_FILE, self.BRANCH_MISSING_METRIC
+            )
+            in caplog.text
+        )
+        assert 0 == ret