Repository: iterative/dvc

Issue:
Number of already present files should be reflected in push progress
Hey guys, IMHO the number of pushed files should also include the files already found in the remote cache.

For example, when I upload 2 new files out of my cache of 1000 files, the progress should say 1000/1000 at the end, instead of 2/1000.


Discussion:
Hi @prihoda !

Hm, actually it should say 2/2, reflecting the current operation . Seems like a bug. Are you using s3 as a cloud or something else?
Oh I see. I'm using S3. Here's an example output of dvc push:

```
(1/19873): [##############################] 100% cb/10ad49259da09638c98d275154cb5b
(2/19873): [##############################] 100% 75/185ea327da3198cfccbdb8d6090518
(3/19873): [##############################] 100% f6/e643730066365e81799e5bac82abb5
(4/19873): [##############################] 100% 11/feba806e24a6496cfea625d4df5145
(5/19873): [##############################] 100% 88/73dd540eecc91af7d52d5ebfb909d0
(6/19873): [##############################] 100% 62/4ea5e4da72058fcf31682435677faa
(7/19873): [##############################] 100% 0c/dbcfbb8aaad9fb2d87f800137b8571
(8/19873): [##############################] 100% 99/2ccda7c3f50886eef2cb701ede2187
(9/19873): [##############################] 100% 6a/da3b667f67445b553ac688f9916fde
(10/19873): [##############################] 100% 40/50b80c0055f604dfda4319d9e72173
(11/19873): [##############################] 100% 57/1145776b993440ffef9813998b00fd
(12/19873): [##############################] 100% ac/7de0e2bd167fe7424085315b061a86
(13/19873): [##############################] 100% 2a/9745aa66aa0cee9ee89eb05818656a
(14/19873): [##############################] 100% 61/9c58f265e8a89d065c95f07b564dc8
(15/19873): [##############################] 100% 59/c24ce3fe6ba07c3950a24c7db7c3a2
```
Yep, looks like a bug. I'll look into it today. Thank you.

Resolving patch:
diff --git a/dvc/cloud/base.py b/dvc/cloud/base.py
--- a/dvc/cloud/base.py
+++ b/dvc/cloud/base.py
@@ -221,3 +221,9 @@ def connect(self):
 
     def disconnect(self):
         pass
+
+    def __enter__(self):
+        self.connect()
+
+    def __exit__(self, type, value, tb):
+        self.disconnect()
diff --git a/dvc/cloud/data_cloud.py b/dvc/cloud/data_cloud.py
--- a/dvc/cloud/data_cloud.py
+++ b/dvc/cloud/data_cloud.py
@@ -11,6 +11,7 @@
 from dvc.cloud.ssh import DataCloudSSH
 from dvc.cloud.local import DataCloudLOCAL
 from dvc.cloud.base import DataCloudBase, CloudSettings
+from dvc.cloud.base import STATUS_MODIFIED, STATUS_NEW, STATUS_DELETED
 
 
 class DataCloud(object):
@@ -99,47 +100,54 @@ def _collect(self, cloud, targets, jobs, local):
 
         return collected
 
-    def _map_targets(self, func, targets, jobs, collect_local=False, collect_cloud=False, remote=None):
-        """
-        Process targets as data items in parallel.
-        """
+    def _get_cloud(self, remote):
+        if remote:
+            return self._init_remote(remote)
 
-        if not remote:
-            cloud = self._cloud
-        else:
-            cloud = self._init_remote(remote)
+        return self._cloud
 
+    def _filter(self, func, status, targets, jobs, remote):
+        cloud = self._get_cloud(remote)
         if not cloud:
-            return
-
-        cloud.connect()
-
-        collected = set()
-        if collect_local:
-            collected |= self._collect(cloud, targets, jobs, True)
-        if collect_cloud:
-            collected |= self._collect(cloud, targets, jobs, False)
+            return []
 
-        ret = map_progress(getattr(cloud, func), list(collected), jobs)
+        with cloud:
+            filtered = []
+            for t, s in self._status(cloud, targets, jobs):
+                if s == STATUS_MODIFIED or s == status:
+                    filtered.append(t)
 
-        cloud.disconnect()
-
-        return ret
+            return map_progress(getattr(cloud, func), filtered, jobs)
 
     def push(self, targets, jobs=1, remote=None):
         """
         Push data items in a cloud-agnostic way.
         """
-        return self._map_targets('push', targets, jobs, collect_local=True, remote=remote)
+        return self._filter('push', STATUS_NEW, targets, jobs, remote)
 
     def pull(self, targets, jobs=1, remote=None):
         """
         Pull data items in a cloud-agnostic way.
         """
-        return self._map_targets('pull', targets, jobs, collect_cloud=True, remote=remote)
+        return self._filter('pull', STATUS_DELETED, targets, jobs, remote)
+
+    def _collect_targets(self, cloud, targets, jobs=1):
+        collected = set()
+        collected |= self._collect(cloud, targets, jobs, True)
+        collected |= self._collect(cloud, targets, jobs, False)
+        return list(collected)
+
+    def _status(self, cloud, targets, jobs=1):
+        collected = self._collect_targets(cloud, targets, jobs)
+        return map_progress(cloud.status, collected, jobs)
 
     def status(self, targets, jobs=1, remote=None):
         """
         Check status of data items in a cloud-agnostic way.
         """
-        return self._map_targets('status', targets, jobs, True, True, remote=remote)
+        cloud = self._get_cloud(remote)
+        if not cloud:
+            return []
+
+        with cloud:
+            return self._status(cloud, targets, jobs)


Test patch: