Repository: iterative/dvc

Issue:
dvc run is supper slow for directories
It should be related to one of the recent changes since I haven't observer such issue before.

For an output directory with 25K files, it takes hours to save state. It looks like dvc saves\rewrites state file for each of the output files. 


Discussion:
WIll take a look today.
Ah, this is a direct cause of that 'state' bug. Will be fixed with #545 . Already working on debugging that.

Resolving patch:
diff --git a/dvc/output.py b/dvc/output.py
--- a/dvc/output.py
+++ b/dvc/output.py
@@ -86,7 +86,7 @@ def save(self):
         if not os.path.isfile(self.path) and not os.path.isdir(self.path):
             raise CmdOutputIsNotFileOrDirError(self.rel_path)
 
-        self.md5 = self.project.state.update(self.path).md5
+        self.md5 = self.project.state.update(self.path)
 
     def dumpd(self, cwd):
         return {
@@ -157,8 +157,8 @@ def loads_from(cls, project, s_list, use_cache=False, cwd=os.curdir):
         return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list]
 
     def _changed_cache(self, cache):
-        state = self.project.state.update(cache)
-        if state.md5 != self.project.cache.path_to_md5(cache):
+        md5 = self.project.state.update(cache)
+        if md5 != self.project.cache.path_to_md5(cache):
             self.project.logger.debug('Corrupted cache file {}'.format(cache))
             self.project._remove_cache_file(cache)
             return True
@@ -303,8 +303,10 @@ def _collect_dir(self):
                 path = os.path.join(root, fname)
                 relpath = os.path.relpath(path, self.path)
 
-                state = self.project.state.update(path)
-                dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: state.md5})
+                md5 = self.project.state.update(path, dump=False)
+                dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5})
+
+        self.project.state.dump()
 
         return dir_info
 
diff --git a/dvc/state.py b/dvc/state.py
--- a/dvc/state.py
+++ b/dvc/state.py
@@ -1,5 +1,5 @@
 import os
-import tinydb
+import json
 from checksumdir import dirhash
 
 from dvc.system import System
@@ -11,30 +11,25 @@
 class StateEntry(object):
     PARAM_MTIME = 'mtime'
     PARAM_MD5 = 'md5'
-    PARAM_INODE = 'inode'
 
-    def __init__(self, md5, mtime, inode):
+    def __init__(self, md5, mtime):
         self.mtime = mtime
         self.md5 = md5
-        self.inode = inode
 
-    def update(self, md5, mtime, inode):
+    def update(self, md5, mtime):
         self.mtime = mtime
         self.md5 = md5
-        self.inode = inode
 
     @staticmethod
     def loadd(d):
         mtime = d[StateEntry.PARAM_MTIME]
         md5 = d[StateEntry.PARAM_MD5]
-        inode = d[StateEntry.PARAM_INODE]
-        return StateEntry(md5, mtime, inode)
+        return StateEntry(md5, mtime)
 
     def dumpd(self):
         return {
             self.PARAM_MD5: self.md5,
             self.PARAM_MTIME: self.mtime,
-            self.PARAM_INODE: self.inode
         }
 
 
@@ -49,8 +44,7 @@ def __init__(self, root_dir, dvc_dir):
         self.root_dir = root_dir
         self.dvc_dir = dvc_dir
         self.state_file = os.path.join(dvc_dir, self.STATE_FILE)
-        self._db = tinydb.TinyDB(self.state_file)
-        self._q = tinydb.Query()
+        self._db = {}
 
     @staticmethod
     def init(root_dir, dvc_dir):
@@ -63,38 +57,42 @@ def compute_md5(self, path):
             return file_md5(path)[0]
 
     def changed(self, path, md5):
-        state = self.update(path)
-        return state.md5 != md5
+        return self.update(path) != md5
 
-    def update(self, path):
+    def load(self):
+        with open(self.state_file, 'r') as fd:
+            self._db = json.load(fd)
+
+    def dump(self):
+        with open(self.state_file, 'w+') as fd:
+            json.dump(self._db, fd)
+
+    def update(self, path, dump=True):
         mtime = os.path.getmtime(path)
         inode = System.inode(path)
 
-        state = self._get(inode, mtime)
-        if state:
-            return state
+        md5 = self._get(inode, mtime)
+        if md5:
+            return md5
 
         md5 = self.compute_md5(path)
-        state = StateEntry(md5, mtime, inode)
+        state = StateEntry(md5, mtime)
         d = state.dumpd()
-        if self._db.contains(self._q.inode == inode):
-            self._db.update(d, self._q.inode == inode)
-        else:
-            self._db.insert(d)
+        self._db[inode] = d
 
-        return state
+        if dump:
+            self.dump()
+
+        return md5
 
     def _get(self, inode, mtime):
-        d_list = self._db.search(self._q.inode == inode)
-        if not len(d_list):
+        d = self._db.get(inode, None)
+        if not d:
             return None
 
-        if len(d_list) > 1:
-            raise StateDuplicateError()
-
-        state = StateEntry.loadd(d_list[0])
-        if mtime == state.mtime and inode == state.inode:
-            return state
+        state = StateEntry.loadd(d)
+        if mtime == state.mtime:
+            return state.md5
 
         return None
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -14,7 +14,6 @@
     "configobj>=5.0.6",
     "networkx>=1.11",
     "pyyaml>=3.12",
-    "tinydb>=3.7.0",
     "gitpython>=2.1.8",
     "ntfsutils>=0.1.3",
     "checksumdir>=1.1.4",


Test patch:
diff --git a/tests/test_checkout.py b/tests/test_checkout.py
--- a/tests/test_checkout.py
+++ b/tests/test_checkout.py
@@ -37,10 +37,6 @@ def test(self):
 
         self.dvc.checkout()
 
-        #FIXME debug
-        import time
-        os.system('cat .dvc/state')
-        print('TIME {}'.format(time.time()))
         self.assertFalse(os.path.isfile(self.FOO))
         self.assertFalse(os.path.isfile(cache))
 
diff --git a/tests/test_state.py b/tests/test_state.py
--- a/tests/test_state.py
+++ b/tests/test_state.py
@@ -1,5 +1,6 @@
 import os
 import stat
+import time
 
 from dvc.system import System
 from dvc.state import State, StateEntry
@@ -17,12 +18,15 @@ def test_update(self):
 
         state = State(self.dvc.root_dir, self.dvc.dvc_dir)
 
-        state.update(path)
-        entry = state.get(path)
-        self.assertIsInstance(entry, StateEntry)
-        self.assertEqual(entry.md5, md5)
-        self.assertEqual(entry.mtime, mtime)
-        self.assertEqual(entry.inode, inode)
+        entry_md5 = state.update(path)
+        self.assertEqual(entry_md5, md5)
+
+        # Sleep some time to simulate realistic behavior.
+        # Some filesystems have a bad date resolution for
+        # mtime(i.e. 1sec for HFS) that cause problems with
+        # our 'state' system not being able to distinguish
+        # files that were modified within that delta.
+        time.sleep(1)        
 
         os.chmod(path, stat.S_IWRITE)
         os.unlink(path)
@@ -33,8 +37,5 @@ def test_update(self):
         mtime = os.path.getmtime(path)
         inode = System.inode(path)
 
-        entry = state.update(path)
-        self.assertIsInstance(entry, StateEntry)
-        self.assertEqual(entry.md5, md5)
-        self.assertEqual(entry.mtime, mtime)
-        self.assertEqual(entry.inode, inode)
+        entry_md5 = state.update(path)
+        self.assertEqual(entry_md5, md5)