Repository: iterative/dvc

Issue:
Re-compute of md5 for large files.
**Please provide information about your setup**
DVC v0.40.1, installed via Pipenv on Ubuntu 16.

See also #1659, which was a similar issue but for large directories.

It seems there is a bug which causes the md5 to be re-computed for large files.  @efiop was able to find a reproducible example for it.  From discord:

```bash
#!/bin/bash                                  
                                             
set -e                                       
set -x                                       
                                             
rm -rf mytest                                
mkdir mytest                                 
cd mytest                                    
                                             
REMOTE=$(pwd)/myremote                       
mkdir $REMOTE                                
                                             
mkdir myrepo                                 
cd myrepo                                    
git init                                     
dvc init                                     
                                             
dvc remote add myremote $REMOTE -d           
                                             
dd if=/dev/urandom of=data  bs=1M  count=1111
                                             
dvc add data                                 
                                             
dvc status                                   
                                             
dvc status                                   
                                             
dvc push                                     
                                             
rm -rf data                                  
rm -rf .dvc/cache                            
                                             
dvc status                                   
                                             
dvc pull                                     
                                             
dvc status                                   
                                             
dvc status                                   
```

> Even on pull it computes md5 of the same file twice. And then again on status after it once. Interesting.

Discussion:

Patch:
diff --git a/dvc/remote/base.py b/dvc/remote/base.py
index 64f8bae6c..7ab579349 100644
--- a/dvc/remote/base.py
+++ b/dvc/remote/base.py
@@ -592,6 +592,7 @@ class RemoteBASE(object):
 
         self.link(cache_info, path_info)
         self.state.save_link(path_info)
+        self.state.save(path_info, checksum)
         if progress_callback:
             progress_callback.update(path_info.url)
 
@@ -613,22 +614,24 @@ class RemoteBASE(object):
         entry_info = copy(path_info)
         for entry in dir_info:
             relpath = entry[self.PARAM_RELPATH]
-            checksum = entry[self.PARAM_CHECKSUM]
-            entry_cache_info = self.checksum_to_path_info(checksum)
+            entry_checksum = entry[self.PARAM_CHECKSUM]
+            entry_cache_info = self.checksum_to_path_info(entry_checksum)
             entry_info.url = self.ospath.join(path_info.url, relpath)
             entry_info.path = self.ospath.join(path_info.path, relpath)
 
-            entry_checksum_info = {self.PARAM_CHECKSUM: checksum}
+            entry_checksum_info = {self.PARAM_CHECKSUM: entry_checksum}
             if self.changed(entry_info, entry_checksum_info):
                 if self.exists(entry_info):
                     self.safe_remove(entry_info, force=force)
                 self.link(entry_cache_info, entry_info)
+                self.state.save(entry_info, entry_checksum)
             if progress_callback:
                 progress_callback.update(entry_info.url)
 
         self._remove_redundant_files(path_info, dir_info, force)
 
         self.state.save_link(path_info)
+        self.state.save(path_info, checksum)
 
     def _remove_redundant_files(self, path_info, dir_info, force):
         existing_files = set(


Test patch:
diff --git a/tests/func/test_add.py b/tests/func/test_add.py
index fb54d7ef7..6b0610d01 100644
--- a/tests/func/test_add.py
+++ b/tests/func/test_add.py
@@ -268,6 +268,15 @@ class TestShouldUpdateStateEntryForFileAfterAdd(TestDvc):
             self.assertEqual(ret, 0)
             self.assertEqual(file_md5_counter.mock.call_count, 1)
 
+            os.rename(self.FOO, self.FOO + ".back")
+            ret = main(["checkout"])
+            self.assertEqual(ret, 0)
+            self.assertEqual(file_md5_counter.mock.call_count, 1)
+
+            ret = main(["status"])
+            self.assertEqual(ret, 0)
+            self.assertEqual(file_md5_counter.mock.call_count, 1)
+
 
 class TestShouldUpdateStateEntryForDirectoryAfterAdd(TestDvc):
     def test(self):
@@ -291,6 +300,15 @@ class TestShouldUpdateStateEntryForDirectoryAfterAdd(TestDvc):
             self.assertEqual(ret, 0)
             self.assertEqual(file_md5_counter.mock.call_count, 3)
 
+            os.rename(self.DATA_DIR, self.DATA_DIR + ".back")
+            ret = main(["checkout"])
+            self.assertEqual(ret, 0)
+            self.assertEqual(file_md5_counter.mock.call_count, 3)
+
+            ret = main(["status"])
+            self.assertEqual(ret, 0)
+            self.assertEqual(file_md5_counter.mock.call_count, 3)
+
 
 class TestAddCommit(TestDvc):
     def test(self):