Repository: iterative/dvc

Issue:
Possible consistency issue when locking and unlocking changed DVC files
Hi guys, 

just wondering whether this could be a problem: After a dvc file content is changed, the md5 checksum of the dvc file can be tricked by locking and unlocking the file, making it seem like the file was not changed. Should dvc refuse to lock an inconsistent file to avoid unintentionally hiding inconsistencies, or would that just provide more headache to the users? 






Discussion:
Or maybe locking and unlocking shouldn't change the md5 checksum. The files won't be shown in `dvc status` anyway so the inconsistency wouldn't be a problem.
Hi @prihoda !

Great catch, as always! The issue is caused by `Stage.dumpd()` method, that is re-computing dvc file checksum even if you simply reload your dvc file, which is not right. It should leave dvc file checksum the way it is, and only compute it when there was an actual `run` performed. You are right, we should exclude unrelated parameters from the dvc file dict, when computing a checksum for it(e.g. `locked`, `metrics`, etc), since they don't affect pipeline or data status. Thank you for the amazing feedback! We will fix this shortly.

Resolving patch:
diff --git a/dvc/stage.py b/dvc/stage.py
--- a/dvc/stage.py
+++ b/dvc/stage.py
@@ -131,13 +131,10 @@ def is_stage_file(path):
         return Stage.is_stage_filename(path)
 
     def changed_md5(self):
-        md5 = self.dumpd().get(self.PARAM_MD5, None)
+        md5 = self._get_md5()
+        assert md5 is not None
 
-        # backward compatibility
-        if self.md5 is None:
-            return False
-
-        if self.md5 and md5 and self.md5 == md5:
+        if self.md5 == md5:
             return False
 
         return True
@@ -371,24 +368,22 @@ def load(project, fname):
             return Stage.loadd(project, yaml.safe_load(fd) or dict(), fname)
 
     def dumpd(self):
-        deps = [x.dumpd() for x in self.deps]
-        outs = [x.dumpd() for x in self.outs]
-
         ret = {}
+
         if self.cmd is not None:
             ret[Stage.PARAM_CMD] = self.cmd
 
-        if len(deps):
-            ret[Stage.PARAM_DEPS] = deps
+        if len(self.deps):
+            ret[Stage.PARAM_DEPS] = [d.dumpd() for d in self.deps]
+
+        if len(self.outs):
+            ret[Stage.PARAM_OUTS] = [o.dumpd() for o in self.outs]
 
-        if len(outs):
-            ret[Stage.PARAM_OUTS] = outs
+        ret[Stage.PARAM_MD5] = self.md5
 
         if self.locked:
             ret[Stage.PARAM_LOCKED] = self.locked
 
-        ret[Stage.PARAM_MD5] = dict_md5(ret)
-
         return ret
 
     def dump(self, fname=None):
@@ -405,6 +400,23 @@ def dump(self, fname=None):
 
         self.project._files_to_git_add.append(os.path.relpath(fname))
 
+    def _get_md5(self):
+        from dvc.output.local import OutputLOCAL
+
+        # NOTE: excluding parameters that don't affect the state of the
+        # pipeline. Not excluding OutputLOCAL.PARAM_CACHE, because if
+        # it has changed, we might not have that output in our cache.
+        exclude = [self.PARAM_LOCKED,
+                   OutputLOCAL.PARAM_METRIC]
+
+        d = self.dumpd()
+
+        # NOTE: removing md5 manually in order to not affect md5s in deps/outs
+        if self.PARAM_MD5 in d.keys():
+            del d[self.PARAM_MD5]
+
+        return dict_md5(d, exclude)
+
     def save(self):
         for dep in self.deps:
             dep.save()
@@ -412,6 +424,8 @@ def save(self):
         for out in self.outs:
             out.save()
 
+        self.md5 = self._get_md5()
+
     def _check_missing_deps(self):
         missing = []
         for dep in self.deps:
diff --git a/dvc/utils.py b/dvc/utils.py
--- a/dvc/utils.py
+++ b/dvc/utils.py
@@ -68,8 +68,31 @@ def bytes_md5(byts):
     return hasher.hexdigest()
 
 
-def dict_md5(d):
-    byts = json.dumps(d, sort_keys=True).encode('utf-8')
+def dict_filter(d, exclude=[]):
+    """
+    Exclude specified keys from a nested dict
+    """
+
+    if isinstance(d, list):
+        ret = []
+        for e in d:
+            ret.append(dict_filter(e, exclude))
+        return ret
+    elif isinstance(d, dict):
+        ret = {}
+        for k, v in d.items():
+            assert isinstance(k, str)
+            if k in exclude:
+                continue
+            ret[k] = dict_filter(v, exclude)
+        return ret
+
+    return d
+
+
+def dict_md5(d, exclude=[]):
+    filtered = dict_filter(d, exclude)
+    byts = json.dumps(filtered, sort_keys=True).encode('utf-8')
     return bytes_md5(byts)
 
 


Test patch:
diff --git a/tests/test_repro.py b/tests/test_repro.py
--- a/tests/test_repro.py
+++ b/tests/test_repro.py
@@ -361,15 +361,54 @@ def test(self):
         self.assertEqual(len(stages), 1)
 
         self.dvc.lock_stage(file1_stage)
-
         stages = self.dvc.reproduce(file1_stage)
         self.assertEqual(len(stages), 0)
 
         self.dvc.lock_stage(file1_stage, unlock=True)
-
         stages = self.dvc.reproduce(file1_stage)
         self.assertEqual(len(stages), 1)
 
+class TestReproLockedUnchanged(TestRepro):
+    def test(self):
+        """
+        Check that locking/unlocking doesn't affect stage state
+        """
+        self.dvc.lock_stage(self.file1_stage)
+        stages = self.dvc.reproduce(self.file1_stage)
+        self.assertEqual(len(stages), 0)
+
+        self.dvc.lock_stage(self.file1_stage, unlock=True)
+        stages = self.dvc.reproduce(self.file1_stage)
+        self.assertEqual(len(stages), 0)
+
+
+class TestReproMetricsAddUnchanged(TestDvc):
+    def test(self):
+        """
+        Check that adding/removing metrics doesn't affect stage state
+        """
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        self.assertTrue(stages[0] is not None)
+
+        file1 = 'file1'
+        file1_stage = file1 + '.dvc'
+        self.dvc.run(fname=file1_stage,
+                     outs_no_cache=[file1],
+                     deps=[self.FOO, self.CODE],
+                     cmd='python {} {} {}'.format(self.CODE, self.FOO, file1))
+
+        stages = self.dvc.reproduce(file1_stage)
+        self.assertEqual(len(stages), 0)
+
+        self.dvc.metrics_add(file1)
+        stages = self.dvc.reproduce(file1_stage)
+        self.assertEqual(len(stages), 0)
+
+        self.dvc.metrics_remove(file1)
+        stages = self.dvc.reproduce(file1_stage)
+        self.assertEqual(len(stages), 0)
+
 
 class TestReproPhony(TestReproChangedData):
     def test(self):
diff --git a/tests/test_stage.py b/tests/test_stage.py
--- a/tests/test_stage.py
+++ b/tests/test_stage.py
@@ -62,3 +62,31 @@ def test_list(self):
         l[1][OutputLOCAL.PARAM_CACHE] = False
         d = {Stage.PARAM_OUTS: l}
         Stage.validate(d)
+
+
+class TestReload(TestDvc):
+    def test(self):
+        import yaml
+        stages = self.dvc.add(self.FOO)
+        self.assertEqual(len(stages), 1)
+        stage = stages[0]
+        self.assertTrue(stage is not None)
+
+        with open(stage.relpath, 'r') as fobj:
+            d = yaml.safe_load(fobj)
+
+        # NOTE: checking that reloaded stage didn't change its checksum
+        md5 = '11111111111111111111111111111111'
+        d[stage.PARAM_MD5] = md5
+
+        with open(stage.relpath, 'w') as fobj:
+            yaml.safe_dump(d, fobj, default_flow_style=False)
+
+        stage = Stage.load(self.dvc, stage.relpath)
+        self.assertTrue(stage is not None)
+        stage.dump()
+
+        with open(stage.relpath, 'r') as fobj:
+            d = yaml.safe_load(fobj)
+
+        self.assertEqual(d[stage.PARAM_MD5], md5)
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -34,3 +34,20 @@ def test_file_md5_crlf(self):
             fd.write(b'a\r\nb\r\nc')
 
         self.assertEqual(utils.file_md5('cr')[0], utils.file_md5('crlf')[0])
+
+    def test_dict_md5(self):
+        d = {'cmd': 'python code.py foo file1',
+             'locked': 'true',
+             'outs': [{'path': 'file1',
+                       'metric': {'type': 'raw'},
+                       'cache': False,
+                       'md5':'acbd18db4cc2f85cedef654fccc4a4d8'}],
+             'deps': [{'path': 'foo',
+                       'md5': 'acbd18db4cc2f85cedef654fccc4a4d8'},
+                      {'path': 'code.py',
+                       'md5': 'd05447644b89960913c7eee5fd776adb'}]}
+
+
+        md5 = '8b263fa05ede6c3145c164829be694b4'
+
+        self.assertEqual(md5, utils.dict_md5(d, exclude=['metric', 'locked']))