Repository: iterative/dvc

Issue:
Error: failed to push data to the cloud
Hi again!

I'm having the following problem:
```bash
$ dvc push -v
Debug: PRAGMA user_version;
Debug: CREATE TABLE IF NOT EXISTS state (inode UNSIGNED INTEGER PRIMARY KEY, mtime TEXT NOT NULL, md5 TEXT NOT NULL, timestamp TEXT NOT NULL)
Debug: CREATE TABLE IF NOT EXISTS state_info (count INTEGER)
Debug: CREATE TABLE IF NOT EXISTS link_state (path TEXT PRIMARY KEY, inode UNSIGNED INTEGER NOT NULL, mtime TEXT NOT NULL)
Debug: INSERT OR IGNORE INTO state_info (count) SELECT 0 WHERE NOT EXISTS (SELECT * FROM state_info)
Debug: PRAGMA user_version = 1;
Preparing to push data to s3://redacted/redacted
[###                           ] 10% Collecting informationDebug: SELECT * from state WHERE inode=7505163933931234253
Debug: UPDATE state SET timestamp = "1539626698545023488" WHERE inode = 7505163933931234253
Debug: SELECT * from state WHERE inode=5912987791212950678
Debug: UPDATE state SET timestamp = "1539626698545638400" WHERE inode = 5912987791212950678
Debug: SELECT * from state WHERE inode=7086292011979178256
Debug: UPDATE state SET timestamp = "1539626698546188800" WHERE inode = 7086292011979178256
Debug: SELECT * from state WHERE inode=2039092171743050023
Debug: UPDATE state SET timestamp = "1539626698546865408" WHERE inode = 2039092171743050023
Debug: SELECT * from state WHERE inode=11883370550676727332
Debug: SELECT count from state_info WHERE rowid=1
Debug: UPDATE state_info SET count = 22 WHERE rowid = 1
Error: Traceback (most recent call last):
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/command/data_sync.py", line 45, in do_run
    with_deps=self.args.with_deps)
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/project.py", line 544, in push
    show_checksums=show_checksums)
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/data_cloud.py", line 108, in push
    show_checksums=show_checksums)
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/remote/local.py", line 621, in push
    checksum_infos = list(filter(func, checksum_infos))
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/remote/local.py", line 620, in func
    return not self.changed_cache(info[self.PARAM_MD5])
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/remote/local.py", line 86, in changed_cache
    if self.state.changed(cache, md5=md5):
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/state.py", line 78, in changed
    actual = self.update(path)
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/state.py", line 269, in update
    return self._do_update(path)[0]
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/state.py", line 248, in _do_update
    assert i == inode
AssertionError

Error: Failed to push data to the cloud:
```

DVC 0.19.11, installed through Pipenv (pip)

Any ideas?


Discussion:

Resolving patch:
diff --git a/dvc/state.py b/dvc/state.py
--- a/dvc/state.py
+++ b/dvc/state.py
@@ -15,10 +15,10 @@ class StateDuplicateError(DvcException):
 
 
 class State(object):
-    VERSION = 1
+    VERSION = 2
     STATE_FILE = 'state'
     STATE_TABLE = 'state'
-    STATE_TABLE_LAYOUT = "inode UNSIGNED INTEGER PRIMARY KEY, " \
+    STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
                          "mtime TEXT NOT NULL, " \
                          "md5 TEXT NOT NULL, " \
                          "timestamp TEXT NOT NULL"
@@ -29,12 +29,15 @@ class State(object):
 
     LINK_STATE_TABLE = 'link_state'
     LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
-                              "inode UNSIGNED INTEGER NOT NULL, " \
+                              "inode INTEGER NOT NULL, " \
                               "mtime TEXT NOT NULL"
 
     STATE_ROW_LIMIT = 10000000
     STATE_ROW_CLEANUP_QUOTA = 50
 
+    MAX_INT = 2**63 - 1
+    MAX_UINT = 2**64 - 2
+
     def __init__(self, project, config):
         self.project = project
         self.dvc_dir = project.dvc_dir
@@ -86,13 +89,38 @@ def _execute(self, cmd):
         Logger.debug(cmd)
         return self.c.execute(cmd)
 
+    def _fetchall(self):
+        ret = self.c.fetchall()
+        Logger.debug("fetched: {}".format(ret))
+        return ret
+
+    def _to_sqlite(self, n):
+        assert n >= 0
+        assert n < self.MAX_UINT
+        # NOTE: sqlite stores unit as signed ints, so maximum uint is 2^63-1
+        # see http://jakegoulding.com/blog/2011/02/06/sqlite-64-bit-integers/
+        if n > self.MAX_INT:
+            ret = -(n - self.MAX_INT)
+        else:
+            ret = n
+        assert self._from_sqlite(ret) == n
+        return ret
+
+    def _from_sqlite(self, n):
+        assert abs(n) <= self.MAX_INT
+        if n < 0:
+            return abs(n) + self.MAX_INT
+        assert n < self.MAX_UINT
+        assert n >= 0
+        return n
+
     def _load(self, empty=False):
         from dvc import VERSION
 
         if not empty:
             cmd = "PRAGMA user_version;"
             self._execute(cmd)
-            ret = self.c.fetchall()
+            ret = self._fetchall()
             assert len(ret) == 1
             assert len(ret[0]) == 1
             assert isinstance(ret[0][0], int)
@@ -165,10 +193,10 @@ def dump(self):
         cmd = "SELECT count from {} WHERE rowid={}"
         self._execute(cmd.format(self.STATE_INFO_TABLE,
                                  self.STATE_INFO_ROW))
-        ret = self.c.fetchall()
+        ret = self._fetchall()
         assert len(ret) == 1
         assert len(ret[0]) == 1
-        count = ret[0][0] + self.inserts
+        count = self._from_sqlite(ret[0][0]) + self.inserts
 
         if count > self.row_limit:
             msg = "Cleaning up state. This might take a while."
@@ -187,14 +215,14 @@ def dump(self):
             cmd = "SELECT COUNT(*) FROM {}"
 
             self._execute(cmd.format(self.STATE_TABLE))
-            ret = self.c.fetchall()
+            ret = self._fetchall()
             assert len(ret) == 1
             assert len(ret[0]) == 1
             count = ret[0][0]
 
         cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
         self._execute(cmd.format(self.STATE_INFO_TABLE,
-                                 count,
+                                 self._to_sqlite(count),
                                  self.STATE_INFO_ROW))
 
         self.db.commit()
@@ -229,16 +257,16 @@ def _do_update(self, path):
         inode = self.inode(path)
 
         cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
-                                                       inode)
+                                                       self._to_sqlite(inode))
 
         self._execute(cmd)
-        ret = self.c.fetchall()
+        ret = self._fetchall()
         if len(ret) == 0:
             md5, info = self._collect(path)
             cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
                   'VALUES ({}, "{}", "{}", "{}")'
             self._execute(cmd.format(self.STATE_TABLE,
-                                     inode,
+                                     self._to_sqlite(inode),
                                      mtime,
                                      md5,
                                      int(nanotime.timestamp(time.time()))))
@@ -247,6 +275,7 @@ def _do_update(self, path):
             assert len(ret) == 1
             assert len(ret[0]) == 4
             i, m, md5, timestamp = ret[0]
+            i = self._from_sqlite(i)
             assert i == inode
             if mtime != m:
                 md5, info = self._collect(path)
@@ -257,13 +286,13 @@ def _do_update(self, path):
                                          mtime,
                                          md5,
                                          int(nanotime.timestamp(time.time())),
-                                         inode))
+                                         self._to_sqlite(inode)))
             else:
                 info = None
                 cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
                 self._execute(cmd.format(self.STATE_TABLE,
                                          int(nanotime.timestamp(time.time())),
-                                         inode))
+                                         self._to_sqlite(inode)))
 
         return (md5, info)
 
@@ -287,7 +316,7 @@ def update_link(self, path):
         cmd = 'REPLACE INTO {}(path, inode, mtime) ' \
               'VALUES ("{}", {}, "{}")'.format(self.LINK_STATE_TABLE,
                                                relpath,
-                                               inode,
+                                               self._to_sqlite(inode),
                                                mtime)
         self._execute(cmd)
 
@@ -297,6 +326,7 @@ def remove_unused_links(self, used):
         self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
         for row in self.c:
             p, i, m = row
+            i = self._from_sqlite(i)
             path = os.path.join(self.root_dir, p)
 
             if path in used:
diff --git a/dvc/system.py b/dvc/system.py
--- a/dvc/system.py
+++ b/dvc/system.py
@@ -178,10 +178,13 @@ class BY_HANDLE_FILE_INFORMATION(Structure):
     @staticmethod
     def inode(path):
         if System.is_unix():
-            return os.lstat(path).st_ino
-
-        # getdirinfo from ntfsutils works on both files and dirs
-        info = System.getdirinfo(path)
-        return hash((info.dwVolumeSerialNumber,
-                     info.nFileIndexHigh,
-                     info.nFileIndexLow))
+            inode = os.lstat(path).st_ino
+        else:
+            # getdirinfo from ntfsutils works on both files and dirs
+            info = System.getdirinfo(path)
+            inode = abs(hash((info.dwVolumeSerialNumber,
+                              info.nFileIndexHigh,
+                              info.nFileIndexLow)))
+        assert inode >= 0
+        assert inode < 2**64
+        return inode


Test patch: