Repository: iterative/dvc

Issue:
state: save file size
This is going to help us combat low-resolution timestamp in some filesystems. We could remove sleep() calls from our tests after this is implemented.



Discussion:

Resolving patch:
diff --git a/dvc/remote/local.py b/dvc/remote/local.py
--- a/dvc/remote/local.py
+++ b/dvc/remote/local.py
@@ -213,7 +213,7 @@ def collect_dir_cache(self, dname):
         dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))
 
         md5 = dict_md5(dir_info) + self.MD5_DIR_SUFFIX
-        if self.changed_cache(md5):
+        if self.changed_cache_file(md5):
             self.dump_dir_cache(md5, dir_info)
 
         return (md5, dir_info)
diff --git a/dvc/state.py b/dvc/state.py
--- a/dvc/state.py
+++ b/dvc/state.py
@@ -15,22 +15,23 @@ class StateDuplicateError(DvcException):
 
 
 class State(object):
-    VERSION = 2
+    VERSION = 3
     STATE_FILE = 'state'
     STATE_TABLE = 'state'
-    STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
-                         "mtime TEXT NOT NULL, " \
-                         "md5 TEXT NOT NULL, " \
-                         "timestamp TEXT NOT NULL"
+    STATE_TABLE_LAYOUT = ("inode INTEGER PRIMARY KEY, "
+                          "mtime TEXT NOT NULL, "
+                          "size TEXT NOT NULL, "
+                          "md5 TEXT NOT NULL, "
+                          "timestamp TEXT NOT NULL")
 
     STATE_INFO_TABLE = 'state_info'
     STATE_INFO_TABLE_LAYOUT = 'count INTEGER'
     STATE_INFO_ROW = 1
 
     LINK_STATE_TABLE = 'link_state'
-    LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
-                              "inode INTEGER NOT NULL, " \
-                              "mtime TEXT NOT NULL"
+    LINK_STATE_TABLE_LAYOUT = ("path TEXT PRIMARY KEY, "
+                               "inode INTEGER NOT NULL, "
+                               "mtime TEXT NOT NULL")
 
     STATE_ROW_LIMIT = 100000000
     STATE_ROW_CLEANUP_QUOTA = 50
@@ -245,17 +246,27 @@ def dump(self):
         self.inserts = 0
 
     @staticmethod
-    def mtime(path):
+    def mtime_and_size(path):
+        size = 0
         mtime = os.path.getmtime(path)
+
         if os.path.isdir(path):
             for root, dirs, files in os.walk(path):
-                for entry in list(dirs) + list(files):
-                    p = os.path.join(root, entry)
+                for dname in dirs:
+                    p = os.path.join(root, dname)
                     m = os.path.getmtime(p)
                     if m > mtime:
                         mtime = m
 
-        return str(int(nanotime.timestamp(mtime)))
+                for fname in files:
+                    p = os.path.join(root, fname)
+                    st = os.stat(p)
+                    size += st.st_size
+                    m = st.st_mtime
+                    if m > mtime:
+                        mtime = m
+
+        return (str(int(nanotime.timestamp(mtime))), size)
 
     @staticmethod
     def inode(path):
@@ -269,7 +280,7 @@ def _do_update(self, path):
         if not os.path.exists(path):
             return (None, None)
 
-        mtime = self.mtime(path)
+        mtime, size = self.mtime_and_size(path)
         inode = self.inode(path)
 
         cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
@@ -279,29 +290,33 @@ def _do_update(self, path):
         ret = self._fetchall()
         if len(ret) == 0:
             md5, info = self._collect(path)
-            cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
-                  'VALUES ({}, "{}", "{}", "{}")'
+            cmd = 'INSERT INTO {}(inode, mtime, size, md5, timestamp) ' \
+                  'VALUES ({}, "{}", "{}", "{}", "{}")'
             self._execute(cmd.format(self.STATE_TABLE,
                                      self._to_sqlite(inode),
                                      mtime,
+                                     size,
                                      md5,
                                      int(nanotime.timestamp(time.time()))))
             self.inserts += 1
         else:
             assert len(ret) == 1
-            assert len(ret[0]) == 4
-            i, m, md5, timestamp = ret[0]
+            assert len(ret[0]) == 5
+            i, m, s, md5, timestamp = ret[0]
             i = self._from_sqlite(i)
             assert i == inode
-            msg = "Inode '{}', mtime '{}', actual mtime '{}'."
-            logger.debug(msg.format(i, m, mtime))
-            if mtime != m:
+            msg = ("Inode '{}', mtime '{}', actual mtime '{}', "
+                   "size '{}', actual size '{}'.")
+            logger.debug(msg.format(i, m, mtime, s, size))
+            if mtime != m or size != s:
                 md5, info = self._collect(path)
-                cmd = 'UPDATE {} SET ' \
-                      'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
-                      'WHERE inode = {}'
+                cmd = ('UPDATE {} SET '
+                       'mtime = "{}", size = "{}", '
+                       'md5 = "{}", timestamp = "{}" '
+                       'WHERE inode = {}')
                 self._execute(cmd.format(self.STATE_TABLE,
                                          mtime,
+                                         size,
                                          md5,
                                          int(nanotime.timestamp(time.time())),
                                          self._to_sqlite(inode)))
@@ -327,7 +342,7 @@ def update_link(self, path):
         if not os.path.exists(path):
             return
 
-        mtime = self.mtime(path)
+        mtime, _ = self.mtime_and_size(path)
         inode = self.inode(path)
         relpath = os.path.relpath(path, self.root_dir)
 
@@ -354,7 +369,7 @@ def remove_unused_links(self, used):
                 continue
 
             inode = self.inode(path)
-            mtime = self.mtime(path)
+            mtime, _ = self.mtime_and_size(path)
 
             if i == inode and m == mtime:
                 logger.debug('Removing \'{}\' as unused link.'.format(path))


Test patch:
diff --git a/tests/test_checkout.py b/tests/test_checkout.py
--- a/tests/test_checkout.py
+++ b/tests/test_checkout.py
@@ -1,6 +1,5 @@
 import os
 import yaml
-import time
 import shutil
 import filecmp
 import collections
@@ -58,8 +57,6 @@ class TestCheckoutCorruptedCacheFile(TestRepro):
     def test(self):
         cache = self.foo_stage.outs[0].cache
 
-        time.sleep(1)
-
         with open(cache, 'a') as fd:
             fd.write('1')
 
@@ -71,8 +68,6 @@ def test(self):
 
 class TestCheckoutCorruptedCacheDir(TestDvc):
     def test(self):
-        time.sleep(1)
-
         # NOTE: using 'copy' so that cache and link don't have same inode
         ret = main(['config', 'cache.type', 'copy'])
         self.assertEqual(ret, 0)
@@ -165,39 +160,31 @@ def test(self):
 class TestCheckoutCleanWorkingDir(CheckoutBase):
     @patch('dvc.prompt.confirm', return_value=True)
     def test(self, mock_prompt):
-        from tests.test_data_cloud import sleep
+        mock_prompt.return_value = True
 
         stages = self.dvc.add(self.DATA_DIR)
         stage = stages[0]
 
-        sleep()
-
         working_dir_change = os.path.join(self.DATA_DIR, 'not_cached.txt')
         with open(working_dir_change, 'w') as f:
             f.write('not_cached')
 
-        sleep()
-
         ret = main(['checkout', stage.relpath])
         self.assertEqual(ret, 0)
         self.assertFalse(os.path.exists(working_dir_change))
 
     @patch('dvc.prompt.confirm', return_value=False)
     def test_force(self, mock_prompt):
-        from tests.test_data_cloud import sleep
+        mock_prompt.return_value = False
 
         stages = self.dvc.add(self.DATA_DIR)
         self.assertEqual(len(stages), 1)
         stage = stages[0]
 
-        sleep()
-
         working_dir_change = os.path.join(self.DATA_DIR, 'not_cached.txt')
         with open(working_dir_change, 'w') as f:
             f.write('not_cached')
 
-        sleep()
-
         ret = main(['checkout', stage.relpath])
         self.assertNotEqual(ret, 0)
 
@@ -208,8 +195,6 @@ def test_force(self, mock_prompt):
 
 class TestCheckoutSelectiveRemove(CheckoutBase):
     def test(self):
-        from tests.test_data_cloud import sleep
-
         # Use copy to test for changes in the inodes
         ret = main(['config', 'cache.type', 'copy'])
         self.assertEqual(ret, 0)
@@ -219,17 +204,11 @@ def test(self):
         stage = stages[0]
         staged_files = self.outs_info(stage)
 
-        sleep()
-
         os.remove(staged_files[0].path)
 
-        sleep()
-
         ret = main(['checkout', '--force', stage.relpath])
         self.assertEqual(ret, 0)
 
-        sleep()
-
         checkedout_files = self.outs_info(stage)
 
         self.assertEqual(len(staged_files), len(checkedout_files))
diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py
--- a/tests/test_data_cloud.py
+++ b/tests/test_data_cloud.py
@@ -2,7 +2,6 @@
 from subprocess import check_output, Popen
 from unittest import SkipTest
 import os
-import time
 import uuid
 import shutil
 import getpass
@@ -151,15 +150,6 @@ def get_azure_url():
     return 'azure://{}/{}'.format(container_name, str(uuid.uuid4()))
 
 
-def sleep():
-    # Sleep some time to simulate realistic behavior.
-    # Some filesystems have a bad date resolution for
-    # mtime(i.e. 1sec for HFS) that cause problems with
-    # our 'state' system not being able to distinguish
-    # files that were modified within that delta.
-    time.sleep(1)
-
-
 class TestDataCloud(TestDvc):
     def _test_cloud(self, config, cl):
         cloud = DataCloud(self.dvc, config=config)
@@ -242,7 +232,6 @@ def _test_cloud(self):
 
         with self.cloud.project.state:
             # Check status
-            sleep()
             status = self.cloud.status([info], show_checksums=True)
             self.assertEqual([(md5, STATUS_NEW)], status)
 
@@ -257,7 +246,6 @@ def _test_cloud(self):
             self.cloud.push([info_dir])
             self.assertTrue(os.path.isfile(cache_dir))
 
-            sleep()
             status = self.cloud.status([info], show_checksums=True)
             self.assertEqual([(md5, STATUS_OK)], status)
 
@@ -265,7 +253,6 @@ def _test_cloud(self):
             self.assertTrue((md5_dir, STATUS_OK) in status_dir)
 
             # Remove and check status
-            sleep()
             shutil.rmtree(self.dvc.cache.local.cache_dir)
 
             status = self.cloud.status([info], show_checksums=True)
@@ -284,7 +271,6 @@ def _test_cloud(self):
             self.cloud.pull([info_dir])
             self.assertTrue(os.path.isfile(cache_dir))
 
-            sleep()
             status = self.cloud.status([info], show_checksums=True)
             self.assertEqual([(md5, STATUS_OK)], status)
 
@@ -397,7 +383,6 @@ def _test_cloud(self, remote=None):
         cache_dir = stage_dir.outs[0].cache
 
         # FIXME check status output
-        sleep()
         self.main(['status', '-c', '--show-checksums'] + args)
 
         self.main(['push'] + args)
@@ -405,12 +390,10 @@ def _test_cloud(self, remote=None):
         self.assertTrue(os.path.isfile(cache))
         self.assertTrue(os.path.isfile(cache_dir))
 
-        sleep()
         self.main(['status', '-c', '--show-checksums'] + args)
 
         shutil.rmtree(self.dvc.cache.local.cache_dir)
 
-        sleep()
         self.main(['status', '-c', '--show-checksums'] + args)
 
         self.main(['fetch'] + args)
@@ -425,7 +408,6 @@ def _test_cloud(self, remote=None):
         self.assertTrue(os.path.isfile(self.FOO))
         self.assertTrue(os.path.isdir(self.DATA_DIR))
 
-        sleep()
         with open(cache, 'r') as fd:
             self.assertEqual(fd.read(), self.FOO_CONTENTS)
         self.assertTrue(os.path.isfile(cache_dir))
@@ -437,10 +419,8 @@ def _test_cloud(self, remote=None):
         shutil.move(self.dvc.cache.local.cache_dir,
                     self.dvc.cache.local.cache_dir + '.back')
 
-        sleep()
         self.main(['fetch'] + args)
 
-        sleep()
         self.main(['pull'] + args)
         self.assertTrue(os.path.exists(cache))
         self.assertTrue(os.path.isfile(cache))
diff --git a/tests/test_repro.py b/tests/test_repro.py
--- a/tests/test_repro.py
+++ b/tests/test_repro.py
@@ -26,7 +26,6 @@
 from tests.test_data_cloud import _should_test_aws, TEST_AWS_REPO_BUCKET
 from tests.test_data_cloud import _should_test_gcp, TEST_GCP_REPO_BUCKET
 from tests.test_data_cloud import _should_test_ssh, _should_test_hdfs
-from tests.test_data_cloud import sleep
 from tests.utils.httpd import StaticFileServer
 
 
@@ -64,8 +63,6 @@ def test(self):
         self.dir_stage = stages[0]
         self.assertTrue(self.dir_stage is not None)
 
-        sleep()
-
         self.file1 = 'file1'
         self.file1_stage = self.file1 + '.dvc'
         self.dvc.run(fname=self.file1_stage,
@@ -80,8 +77,6 @@ def test(self):
         os.unlink(self.DATA)
         shutil.copyfile(self.FOO, self.DATA)
 
-        sleep()
-
         stages = self.dvc.reproduce(self.file1_stage)
         self.assertEqual(len(stages), 2)
         self.assertTrue(filecmp.cmp(self.file1, self.FOO, shallow=False))
@@ -93,20 +88,14 @@ def test(self):
         self.assertEqual(len(stages), 1)
         self.assertTrue(stages[0] is not None)
 
-        sleep()
-
         stages = self.dvc.add(self.DATA_SUB)
         self.assertEqual(len(stages), 1)
         self.assertTrue(stages[0] is not None)
 
-        sleep()
-
         stage = self.dvc.run(fname='Dvcfile',
                              deps=[self.DATA, self.DATA_SUB])
         self.assertTrue(stage is not None)
 
-        sleep()
-
         file1 = 'file1'
         file1_stage = file1 + '.dvc'
         stage = self.dvc.run(fname=file1_stage,
@@ -117,13 +106,9 @@ def test(self):
                                                           file1))
         self.assertTrue(stage is not None)
 
-        sleep()
-
         os.unlink(self.DATA)
         shutil.copyfile(self.FOO, self.DATA)
 
-        sleep()
-
         stages = self.dvc.reproduce(file1_stage)
         self.assertEqual(len(stages), 2)
 
@@ -287,8 +272,6 @@ def test(self):
                                                           file2))
         self.assertTrue(stage is not None)
 
-        sleep()
-
         with open(code1, 'a') as fobj:
             fobj.write('\n\n')
 
@@ -521,14 +504,10 @@ def test(self):
         dir_name = 'dir'
         dir_code = 'dir_code.py'
 
-        sleep()
-
         with open(dir_code, 'w+') as fd:
             fd.write("import os; import sys; import shutil; "
                      "shutil.copytree(sys.argv[1], sys.argv[2])")
 
-        sleep()
-
         stage = self.dvc.run(outs=[dir_name],
                              deps=[self.DATA_DIR, dir_code],
                              cmd="python {} {} {}".format(dir_code,
@@ -539,28 +518,21 @@ def test(self):
         stages = self.dvc.reproduce(stage.path)
         self.assertEqual(len(stages), 0)
 
-        sleep()
-
         with open(self.DATA_SUB, 'a') as fd:
             fd.write('add')
 
-        sleep()
-
         stages = self.dvc.reproduce(stage.path)
         self.assertEqual(len(stages), 1)
         self.assertTrue(stages[0] is not None)
-        sleep()
 
         # Check that dvc indeed registers changed output dir
         shutil.move(self.BAR, dir_name)
-        sleep()
         stages = self.dvc.reproduce(stage.path)
         self.assertEqual(len(stages), 1)
         self.assertTrue(stages[0] is not None)
 
         # Check that dvc registers mtime change for the directory.
         System.hardlink(self.DATA_SUB, self.DATA_SUB + '.lnk')
-        sleep()
         stages = self.dvc.reproduce(stage.path)
         self.assertEqual(len(stages), 1)
         self.assertTrue(stages[0] is not None)
@@ -685,8 +657,6 @@ def test(self):
 
         self.write(self.bucket, foo_key, self.FOO_CONTENTS)
 
-        sleep()
-
         import_stage = self.dvc.imp(out_foo_path, 'import')
         self.assertTrue(os.path.exists('import'))
         self.assertTrue(filecmp.cmp('import', self.FOO, shallow=False))
@@ -705,8 +675,6 @@ def test(self):
 
         self.write(self.bucket, foo_key, self.BAR_CONTENTS)
 
-        sleep()
-
         self.assertNotEqual(self.dvc.status(), {})
 
         stages = self.dvc.reproduce(import_stage.path)
diff --git a/tests/test_run.py b/tests/test_run.py
--- a/tests/test_run.py
+++ b/tests/test_run.py
@@ -332,8 +332,6 @@ def test(self):
 
 class TestCmdRunOverwrite(TestDvc):
     def test(self):
-        import time
-
         ret = main(['run',
                     '-d', self.FOO,
                     '-d', self.CODE,
@@ -344,8 +342,6 @@ def test(self):
 
         stage_mtime = os.path.getmtime('out.dvc')
 
-        time.sleep(2)
-
         ret = main(['run',
                     '-d', self.FOO,
                     '-d', self.CODE,
@@ -358,8 +354,6 @@ def test(self):
         self.assertEqual(stage_mtime, os.path.getmtime('out.dvc'))
         stage_mtime = os.path.getmtime('out.dvc')
 
-        time.sleep(2)
-
         ret = main(['run',
                     '-d', self.FOO,
                     '-d', self.CODE,
@@ -374,8 +368,6 @@ def test(self):
         self.assertNotEqual(stage_mtime, os.path.getmtime('out.dvc'))
         stage_mtime = os.path.getmtime('out.dvc')
 
-        time.sleep(2)
-
         ret = main(['run',
                     '--overwrite-dvcfile',
                     '-f', 'out.dvc',
diff --git a/tests/test_state.py b/tests/test_state.py
--- a/tests/test_state.py
+++ b/tests/test_state.py
@@ -1,5 +1,4 @@
 import os
-import time
 
 from dvc.state import State
 from dvc.utils import file_md5
@@ -19,15 +18,8 @@ def test_update(self):
             entry_md5 = state.update(path)
             self.assertEqual(entry_md5, md5)
 
-            # Sleep some time to simulate realistic behavior.
-            # Some filesystems have a bad date resolution for
-            # mtime(i.e. 1sec for HFS) that cause problems with
-            # our 'state' system not being able to distinguish
-            # files that were modified within that delta.
-            time.sleep(1)
-
             os.unlink(path)
-            with open(path, 'w+') as fd:
+            with open(path, 'a') as fd:
                 fd.write('1')
 
             md5 = file_md5(path)[0]