Repository: iterative/dvc

Issue:
dvc run -m/-M: inconsistent behavior with dirs
`dvc run` with `-M` flag doesn't accept directories as metrics whereas `dvc run` with `-m` flag does. This behavior should be unified.

**DVC version:**
- 0.94 (snap package)

**Additional Information :**
- OS: Ubuntu 18.04
- Discord context: https://discord.com/channels/485586884165107732/565699007037571084/717079938506686543
- Possibly related to #2973 

**Steps to reproduce**

1. Initialize git and dvc

2. Create `evaluate.py`:

```
import argparse
import os
import json

parser = argparse.ArgumentParser()
parser.add_argument("--metrics_dir", required=True)
args = parser.parse_args()
metrics_dir = args.metrics_dir

if not os.path.exists(metrics_dir):
    os.makedirs(metrics_dir)

with open(os.path.join(metrics_dir, "metrics1.json"), "w") as f:
    json.dump({"acc": 1.0}, f)

with open(os.path.join(metrics_dir, "metric2.json"), "w") as f:
    json.dump({"acc": 0.5}, f)
```

3. Create new stage with directory as cached metrics output:

```
$ dvc run -f metrics_cached.dvc -d evaluate.py -m metrics_cached  python3 evaluate.py --metrics_dir metrics_cached
Running command:
        python3 evaluate.py --metrics_dir metrics_cached

To track the changes with git, run:

        git add metrics_cached.dvc .gitignore
```

4. Create new stage with directory as not cached metrics output:

```
$ dvc run -f metrics_not_cached.dvc -d evaluate.py -M metrics_not_cached  python3 evaluate.py --metrics_dir metrics_not_cached
Running command:
        python3 evaluate.py --metrics_dir metrics_not_cached
ERROR: failed to run command - directory 'metrics_not_cached' cannot be used as metrics.
```

Both commands should either accept directories as metrics either discard them


Comments:
Also found out that `dvc run --help` explicitly declares support for `dirs` for both of these options:

```
  -m <path>, --metrics <path>
                        Declare output metric file or directory.
  -M <path>, --metrics-no-cache <path>
                        Declare output metric file or directory (do not put
                        into DVC cache).
```
From Discord discussion I assume that `-m`/`-M` flags should not accept directories for now.
@nik123 That's correct! Thank you so much for looking into it! :pray: 

Patch:
diff --git a/dvc/command/run.py b/dvc/command/run.py
--- a/dvc/command/run.py
+++ b/dvc/command/run.py
@@ -133,7 +133,7 @@ def add_parser(subparsers, parent_parser):
         "--metrics",
         action="append",
         default=[],
-        help="Declare output metric file or directory.",
+        help="Declare output metric file.",
         metavar="<path>",
     )
     run_parser.add_argument(
@@ -141,8 +141,7 @@ def add_parser(subparsers, parent_parser):
         "--metrics-no-cache",
         action="append",
         default=[],
-        help="Declare output metric file or directory "
-        "(do not put into DVC cache).",
+        help="Declare output metric file (do not put into DVC cache).",
         metavar="<path>",
     )
     run_parser.add_argument(
diff --git a/dvc/output/base.py b/dvc/output/base.py
--- a/dvc/output/base.py
+++ b/dvc/output/base.py
@@ -246,10 +246,11 @@ def save(self):
 
         self.ignore()
 
+        if self.metric or self.plot:
+            self.verify_metric()
+
         if not self.use_cache:
             self.info = self.save_info()
-            if self.metric or self.plot:
-                self.verify_metric()
             if not self.IS_DEPENDENCY:
                 logger.debug(
                     "Output '%s' doesn't use cache. Skipping saving.", self
diff --git a/dvc/output/local.py b/dvc/output/local.py
--- a/dvc/output/local.py
+++ b/dvc/output/local.py
@@ -81,7 +81,7 @@ def verify_metric(self):
         name = "metrics" if self.metric else "plot"
         if os.path.isdir(path):
             msg = "directory '{}' cannot be used as {}."
-            raise DvcException(msg.format(self.path_info, name))
+            raise IsADirectoryError(msg.format(self.path_info, name))
 
         if not istextfile(path):
             msg = "binary file '{}' cannot be used as {}."


Test patch:
diff --git a/tests/func/test_run_single_stage.py b/tests/func/test_run_single_stage.py
--- a/tests/func/test_run_single_stage.py
+++ b/tests/func/test_run_single_stage.py
@@ -989,3 +989,31 @@ def test_should_raise_on_stage_output(tmp_dir, dvc, run_copy):
 
     with pytest.raises(OutputIsStageFileError):
         run_copy("foo", "name.dvc", single_stage=True)
+
+
+class TestRunDirMetrics:
+    @pytest.fixture(autouse=True)
+    def setup(self, dvc):
+        with open("script.py", "w+") as fobj:
+            fobj.write("import sys\n")
+            fobj.write("import os\n")
+            fobj.write("os.makedirs(sys.argv[1])\n")
+            fobj.write(
+                "with open(os.path.join(sys.argv[1], "
+                "'metrics.json'), 'a+') as fobj:\n"
+            )
+            fobj.write("    fobj.write('foo')\n")
+
+    def test_metrics_dir_cached(self, dvc):
+        with pytest.raises(IsADirectoryError):
+            dvc.run(
+                cmd="python script.py dir", metrics=["dir"], single_stage=True,
+            )
+
+    def test_metrics_dir_not_cached(self, dvc):
+        with pytest.raises(IsADirectoryError):
+            dvc.run(
+                cmd="python script.py dir",
+                metrics_no_cache=["dir"],
+                single_stage=True,
+            )