Repository: iterative/dvc

Issue:
DVC incorrectly ignores outputs
## Bug Report

DVC computes ignored outputs relatively to corresponding `*.dvc` instead of `.dvcignore` location.

### Please provide information about your setup

```
DVC version: 1.10.1+6237a6.mod 
---------------------------------
Platform: Python 3.8.6 on Linux-5.4.79-1-lts-x86_64-with-glibc2.2.5
Supports: All remotes
Cache types: <https://error.dvc.org/no-dvc-cache>
Caches: local
Remotes: https
Workspace directory: ext4 on /dev/sda2
Repo: dvc, git
```

`DVC status -v` output

```
dvc status -v
2020-11-28 16:41:28,063 DEBUG: Check for update is enabled.
2020-11-28 16:41:28,069 DEBUG: fetched: [(3,)]                                  
2020-11-28 16:41:28,073 DEBUG: fetched: [(1404595,)]
2020-11-28 16:41:28,154 ERROR: Path 'fms_unit.csv' is ignored by
.dvcignore:1:/*
------------------------------------------------------------
Traceback (most recent call last):
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/command/status.py", line 58, in run
    recursive=self.args.recursive,
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/repo/__init__.py", line 60, in wrapper
    return f(repo, *args, **kwargs)
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/repo/status.py", line 152, in status
    self, targets, with_deps=with_deps, recursive=recursive
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/repo/status.py", line 36, in _local_status
    return _joint_status(pairs)
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/repo/status.py", line 16, in _joint_status
    for stage, filter_info in pairs:
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/repo/status.py", line 33, in <genexpr>
    for t in targets
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/repo/__init__.py", line 358, in collect_granular
    return [(stage, None) for stage in self.stages]
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/funcy/objects.py", line 28, in __get__
    res = instance.__dict__[self.fget.__name__] = self.fget(instance)
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/repo/__init__.py", line 493, in stages
    return self._collect_stages()
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/repo/__init__.py", line 501, in _collect_stages
    new_stages = self.get_stages(os.path.join(root, file_name))
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/repo/__init__.py", line 290, in get_stages
    return list(dvcfile.stages.values())
  File "/usr/lib/python3.7/_collections_abc.py", line 762, in __iter__
    yield self._mapping[key]
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/stage/loader.py", line 153, in __getitem__
    self.dvcfile, deepcopy(self.stage_data), self.stage_text
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/stage/loader.py", line 164, in load_stage
    stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS) or [])
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/output/__init__.py", line 150, in loadd_from
    desc=desc,
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/output/__init__.py", line 126, in _get
    desc=desc,
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/output/local.py", line 24, in __init__
    super().__init__(stage, path, *args, **kwargs)
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/output/base.py", line 109, in __init__
    self._validate_output_path(path, stage)
  File "/home/anotherbugmaster/.virtualenvs/PTR/lib/python3.7/site-packages/dvc/output/base.py", line 535, in _validate_output_path
    raise cls.IsIgnoredError(check)
dvc.output.base.OutputIsIgnoredError: Path 'fms_unit.csv' is ignored by
.dvcignore:1:/*
------------------------------------------------------------
2020-11-28 16:41:28,214 DEBUG: Analytics is enabled.
2020-11-28 16:41:28,251 DEBUG: Trying to spawn '['daemon', '-q', 'analytics', '/tmp/tmp0m393p29']'
2020-11-28 16:41:28,252 DEBUG: Spawned '['daemon', '-q', 'analytics', '/tmp/tmp0m393p29']'
```


Discussion:

Resolving patch:
diff --git a/dvc/ignore.py b/dvc/ignore.py
--- a/dvc/ignore.py
+++ b/dvc/ignore.py
@@ -91,45 +91,50 @@ def _get_normalize_path(self, dirname, basename):
             path = normalize_file(path)
         return path
 
-    def matches(self, dirname, basename, is_dir=False):
+    def matches(self, dirname, basename, is_dir=False, details: bool = False):
         path = self._get_normalize_path(dirname, basename)
         if not path:
             return False
+
+        if details:
+            return self._ignore_details(path, is_dir)
         return self.ignore(path, is_dir)
 
     def ignore(self, path, is_dir):
+        def matches(pattern, path, is_dir) -> bool:
+            matches_ = bool(pattern.match(path))
+
+            if is_dir:
+                matches_ |= bool(pattern.match(f"{path}/"))
+
+            return matches_
+
         result = False
-        if is_dir:
-            path_dir = f"{path}/"
-            for ignore, pattern in self.ignore_spec:
-                if pattern.match(path) or pattern.match(path_dir):
-                    result = ignore
-        else:
-            for ignore, pattern in self.ignore_spec:
-                if pattern.match(path):
-                    result = ignore
-        return result
 
-    def match_details(self, dirname, basename, is_dir=False):
-        path = self._get_normalize_path(dirname, basename)
-        if not path:
-            return False
-        return self._ignore_details(path, is_dir)
+        for ignore, pattern in self.ignore_spec[::-1]:
+            if matches(pattern, path, is_dir):
+                result = ignore
+                break
+        return result
 
-    def _ignore_details(self, path, is_dir):
+    def _ignore_details(self, path, is_dir: bool):
         result = []
-        for ignore, pattern in zip(self.regex_pattern_list, self.pattern_list):
-            regex = re.compile(ignore[0])
+        for (regex, _), pattern_info in list(
+            zip(self.regex_pattern_list, self.pattern_list)
+        ):
             # skip system pattern
-            if not pattern.file_info:
+            if not pattern_info.file_info:
                 continue
+
+            regex = re.compile(regex)
+
+            matches = bool(regex.match(path))
             if is_dir:
-                path_dir = f"{path}/"
-                if regex.match(path) or regex.match(path_dir):
-                    result.append(pattern.file_info)
-            else:
-                if regex.match(path):
-                    result.append(pattern.file_info)
+                matches |= bool(regex.match(f"{path}/"))
+
+            if matches:
+                result.append(pattern_info.file_info)
+
         return result
 
     def __hash__(self):
@@ -323,13 +328,15 @@ def _outside_repo(self, path):
         return False
 
     def check_ignore(self, target):
+        # NOTE: can only be used in `dvc check-ignore`, see
+        # https://github.com/iterative/dvc/issues/5046
         full_target = os.path.abspath(target)
         if not self._outside_repo(full_target):
             dirname, basename = os.path.split(os.path.normpath(full_target))
             pattern = self._get_trie_pattern(dirname)
             if pattern:
-                matches = pattern.match_details(
-                    dirname, basename, os.path.isdir(full_target)
+                matches = pattern.matches(
+                    dirname, basename, os.path.isdir(full_target), True,
                 )
 
                 if matches:
@@ -339,7 +346,11 @@ def check_ignore(self, target):
     def is_ignored(self, path):
         # NOTE: can't use self.check_ignore(path).match for now, see
         # https://github.com/iterative/dvc/issues/4555
-        return self.is_ignored_dir(path) or self.is_ignored_file(path)
+        if os.path.isfile(path):
+            return self.is_ignored_file(path)
+        if os.path.isdir(path):
+            return self.is_ignored_dir(path)
+        return self.is_ignored_file(path) or self.is_ignored_dir(path)
 
 
 def init(path):
diff --git a/dvc/output/base.py b/dvc/output/base.py
--- a/dvc/output/base.py
+++ b/dvc/output/base.py
@@ -559,8 +559,9 @@ def _validate_output_path(cls, path, stage=None):
             raise cls.IsStageFileError(path)
 
         if stage:
-            check = stage.repo.tree.dvcignore.check_ignore(path)
-            if check.match:
+            abs_path = os.path.join(stage.wdir, path)
+            if stage.repo.tree.dvcignore.is_ignored(abs_path):
+                check = stage.repo.tree.dvcignore.check_ignore(abs_path)
                 raise cls.IsIgnoredError(check)
 
     def _check_can_merge(self, out):


Test patch:
diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py
--- a/tests/dir_helpers.py
+++ b/tests/dir_helpers.py
@@ -332,10 +332,14 @@ def run_copy(tmp_dir, dvc):
     )
 
     def run_copy(src, dst, **run_kwargs):
+        wdir = pathlib.Path(run_kwargs.get("wdir", "."))
+        wdir = pathlib.Path("../" * len(wdir.parts))
+        script_path = wdir / "copy.py"
+
         return dvc.run(
-            cmd=f"python copy.py {src} {dst}",
+            cmd=f"python {script_path} {src} {dst}",
             outs=[dst],
-            deps=[src, "copy.py"],
+            deps=[src, f"{script_path}"],
             **run_kwargs,
         )
 
diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py
--- a/tests/func/test_ignore.py
+++ b/tests/func/test_ignore.py
@@ -1,5 +1,6 @@
 import os
 import shutil
+from pathlib import Path
 
 import pytest
 
@@ -404,7 +405,17 @@ def test_ignore_in_added_dir(tmp_dir, dvc):
 
 
 def test_ignored_output(tmp_dir, scm, dvc, run_copy):
-    tmp_dir.gen({".dvcignore": "*.log", "foo": "foo content"})
+    tmp_dir.gen({".dvcignore": "*.log\n!foo.log", "foo": "foo content"})
 
     with pytest.raises(OutputIsIgnoredError):
-        run_copy("foo", "foo.log", name="copy")
+        run_copy("foo", "abc.log", name="copy")
+
+    run_copy("foo", "foo.log", name="copy")
+
+
+def test_ignored_output_nested(tmp_dir, scm, dvc, run_copy):
+    tmp_dir.gen({".dvcignore": "/*.log", "copy": {"foo": "foo content"}})
+
+    run_copy("foo", "foo.log", name="copy", wdir="copy")
+
+    assert Path("copy/foo.log").exists()
diff --git a/tests/unit/output/test_output.py b/tests/unit/output/test_output.py
--- a/tests/unit/output/test_output.py
+++ b/tests/unit/output/test_output.py
@@ -73,6 +73,10 @@ def test_get_used_cache(exists, expected_message, mocker, caplog):
     stage = mocker.MagicMock()
     mocker.patch.object(stage, "__str__", return_value="stage: 'stage.dvc'")
     mocker.patch.object(stage, "addressing", "stage.dvc")
+    mocker.patch.object(stage, "wdir", ".")
+    mocker.patch.object(
+        stage.repo.tree.dvcignore, "is_ignored", return_value=False,
+    )
     mocker.patch.object(
         stage.repo.tree.dvcignore,
         "check_ignore",