Repository: iterative/dvc

Issue:
dvc add fails with "Unexpected error: datatype mismatch"
```
$ dvc add data/training/events.csv
Saving 'data/training/events.csv' to cache '../dvc-cache'.
Computing md5 for a large file data/training/events.csv. This is only done once.
[##############################] 100% data/training/events.csv
Error: Unexpected error: datatype mismatch
```
DVC version 0.19.8, installed through pip (Pipenv)



Discussion:
Upgraded to 0.19.9 and have the same error
Hi @crmne !

Could you please run that command again but with the `-v` flag added? It will enable more verbose output, so we can see better where the error was raised.

Thanks,
Ruslan
```
Error: Traceback (most recent call last):
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/main.py", line 22, in main
    ret = cmd.run_cmd()
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/command/base.py", line 40, in run_cmd
    return self.run()
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/command/add.py", line 10, in run
    self.project.add(target, recursive=self.args.recursive)
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/project.py", line 173, in add
    stage.save()
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/stage.py", line 393, in save
    out.save()
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/output/local.py", line 113, in save
    self.info = self.project.cache.local.save(self.path_info)
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/remote/local.py", line 325, in save
    return self._save_file(path_info)
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/remote/local.py", line 276, in _save_file
    md5 = self.state.update(path)
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/state.py", line 265, in update
    return self._do_update(path)[0]
  File "/home/ubuntu/.local/share/virtualenvs/9eeOrpBx/lib/python3.7/site-packages/dvc/state.py", line 238, in _do_update
    int(nanotime.timestamp(time.time()))))
sqlite3.IntegrityError: datatype mismatch

Error: Unexpected error: datatype mismatch
```
Thank you! Looks like some value that we are trying to insert doesn't have a right type. Looking into it right now.
I've pushed a patch to my repository that will show us more details of what is going on. Could you please install it and run `dvc add` with `-v` flag once more? E.g.:
```
pip uninstall dvc
pip install git+https://github.com/efiop/dvc.git
```

Resolving patch:
diff --git a/dvc/state.py b/dvc/state.py
--- a/dvc/state.py
+++ b/dvc/state.py
@@ -15,10 +15,10 @@ class StateDuplicateError(DvcException):
 
 
 class State(object):
-    VERSION = 0
+    VERSION = 1
     STATE_FILE = 'state'
     STATE_TABLE = 'state'
-    STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
+    STATE_TABLE_LAYOUT = "inode UNSIGNED INTEGER PRIMARY KEY, " \
                          "mtime TEXT NOT NULL, " \
                          "md5 TEXT NOT NULL, " \
                          "timestamp TEXT NOT NULL"
@@ -29,7 +29,7 @@ class State(object):
 
     LINK_STATE_TABLE = 'link_state'
     LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
-                              "inode INTEGER NOT NULL, " \
+                              "inode UNSIGNED INTEGER NOT NULL, " \
                               "mtime TEXT NOT NULL"
 
     STATE_ROW_LIMIT = 10000000
@@ -82,11 +82,15 @@ def changed(self, path, md5):
 
         return actual.split('.')[0] != md5.split('.')[0]
 
+    def _execute(self, cmd):
+        Logger.debug(cmd)
+        return self.c.execute(cmd)
+
     def _load(self):
         from dvc import VERSION
 
         cmd = "PRAGMA user_version;"
-        self.c.execute(cmd)
+        self._execute(cmd)
         ret = self.c.fetchall()
         assert len(ret) == 1
         assert len(ret[0]) == 1
@@ -106,26 +110,26 @@ def _load(self):
                   "Reformatting to the current version '{}'."
             self.project.logger.warn(msg.format(self.version, self.VERSION))
             cmd = "DROP TABLE IF EXISTS {};"
-            self.c.execute(cmd.format(self.STATE_TABLE))
-            self.c.execute(cmd.format(self.STATE_INFO_TABLE))
-            self.c.execute(cmd.format(self.LINK_STATE_TABLE))
+            self._execute(cmd.format(self.STATE_TABLE))
+            self._execute(cmd.format(self.STATE_INFO_TABLE))
+            self._execute(cmd.format(self.LINK_STATE_TABLE))
 
         # Check that the state file is indeed a database
         cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
-        self.c.execute(cmd.format(self.STATE_TABLE,
-                                  self.STATE_TABLE_LAYOUT))
-        self.c.execute(cmd.format(self.STATE_INFO_TABLE,
-                                  self.STATE_INFO_TABLE_LAYOUT))
-        self.c.execute(cmd.format(self.LINK_STATE_TABLE,
-                                  self.LINK_STATE_TABLE_LAYOUT))
+        self._execute(cmd.format(self.STATE_TABLE,
+                                 self.STATE_TABLE_LAYOUT))
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_TABLE_LAYOUT))
+        self._execute(cmd.format(self.LINK_STATE_TABLE,
+                                 self.LINK_STATE_TABLE_LAYOUT))
 
         cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
               "WHERE NOT EXISTS (SELECT * FROM {})"
-        self.c.execute(cmd.format(self.STATE_INFO_TABLE,
-                                  self.STATE_INFO_TABLE))
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_TABLE))
 
         cmd = "PRAGMA user_version = {};"
-        self.c.execute(cmd.format(self.VERSION))
+        self._execute(cmd.format(self.VERSION))
 
     def load(self):
         retries = 1
@@ -157,8 +161,8 @@ def dump(self):
         assert self.db is not None
 
         cmd = "SELECT count from {} WHERE rowid={}"
-        self.c.execute(cmd.format(self.STATE_INFO_TABLE,
-                                  self.STATE_INFO_ROW))
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 self.STATE_INFO_ROW))
         ret = self.c.fetchall()
         assert len(ret) == 1
         assert len(ret[0]) == 1
@@ -172,24 +176,24 @@ def dump(self):
             delete += int(self.row_limit * (self.row_cleanup_quota/100.))
             cmd = "DELETE FROM {} WHERE timestamp IN (" \
                   "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
-            self.c.execute(cmd.format(self.STATE_TABLE,
-                                      self.STATE_TABLE,
-                                      delete))
+            self._execute(cmd.format(self.STATE_TABLE,
+                                     self.STATE_TABLE,
+                                     delete))
 
-            self.c.execute("VACUUM")
+            self._execute("VACUUM")
 
             cmd = "SELECT COUNT(*) FROM {}"
 
-            self.c.execute(cmd.format(self.STATE_TABLE))
+            self._execute(cmd.format(self.STATE_TABLE))
             ret = self.c.fetchall()
             assert len(ret) == 1
             assert len(ret[0]) == 1
             count = ret[0][0]
 
         cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
-        self.c.execute(cmd.format(self.STATE_INFO_TABLE,
-                                  count,
-                                  self.STATE_INFO_ROW))
+        self._execute(cmd.format(self.STATE_INFO_TABLE,
+                                 count,
+                                 self.STATE_INFO_ROW))
 
         self.db.commit()
         self.c.close()
@@ -225,17 +229,17 @@ def _do_update(self, path):
         cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
                                                        inode)
 
-        self.c.execute(cmd)
+        self._execute(cmd)
         ret = self.c.fetchall()
         if len(ret) == 0:
             md5, info = self._collect(path)
             cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
                   'VALUES ({}, "{}", "{}", "{}")'
-            self.c.execute(cmd.format(self.STATE_TABLE,
-                                      inode,
-                                      mtime,
-                                      md5,
-                                      int(nanotime.timestamp(time.time()))))
+            self._execute(cmd.format(self.STATE_TABLE,
+                                     inode,
+                                     mtime,
+                                     md5,
+                                     int(nanotime.timestamp(time.time()))))
             self.inserts += 1
         else:
             assert len(ret) == 1
@@ -247,17 +251,17 @@ def _do_update(self, path):
                 cmd = 'UPDATE {} SET ' \
                       'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
                       'WHERE inode = {}'
-                self.c.execute(cmd.format(self.STATE_TABLE,
-                                          mtime,
-                                          md5,
-                                          int(nanotime.timestamp(time.time())),
-                                          inode))
+                self._execute(cmd.format(self.STATE_TABLE,
+                                         mtime,
+                                         md5,
+                                         int(nanotime.timestamp(time.time())),
+                                         inode))
             else:
                 info = None
                 cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
-                self.c.execute(cmd.format(self.STATE_TABLE,
-                                          int(nanotime.timestamp(time.time())),
-                                          inode))
+                self._execute(cmd.format(self.STATE_TABLE,
+                                         int(nanotime.timestamp(time.time())),
+                                         inode))
 
         return (md5, info)
 
@@ -283,12 +287,12 @@ def update_link(self, path):
                                                relpath,
                                                inode,
                                                mtime)
-        self.c.execute(cmd)
+        self._execute(cmd)
 
     def remove_unused_links(self, used):
         unused = []
 
-        self.c.execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
+        self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
         for row in self.c:
             p, i, m = row
             path = os.path.join(self.root_dir, p)
@@ -309,4 +313,4 @@ def remove_unused_links(self, used):
 
         for p in unused:
             cmd = 'DELETE FROM {} WHERE path = "{}"'
-            self.c.execute(cmd.format(self.LINK_STATE_TABLE, p))
+            self._execute(cmd.format(self.LINK_STATE_TABLE, p))


Test patch: