Repository: iterative/dvc

Issue:
Use progress bar to show a process of linking many files in a directory
E.g. currently for a directory with 1M files, there will be 1M lines printed with message that says something like "Created 'hardlink' ...". Need to use progress bar to show that in a more compact way.


Discussion:
Adding extra steps to replicate the issue:

```bash
#!/usr/bin/env bash

rm -rf repo
mkdir repo
cd repo
git init
dvc init
git add -A
git commit -m "Initial commit with DVC"

# Create data dir with 1000 files
mkdir data

# for file in {1..1000}; do echo $file > data/$file.txt; done
parallel 'echo {} > data/{}.txt' ::: {1..1000}

# Add the data directory and see a 1000 "Created 'hardlink'..."
dvc add data
```

@efiop, should the progress bar appear after a certain threshold (let's say 30)?

I think this is not that bad, maybe the user would like to `--quiet` the output instead of seeing a progress bar; also, when using `--quiet` the progress bar for:
```
Computing md5 for a large directory data. This is only done once.
[##############################] 100% data
```
Still appears.
@mroutis I think we should have a "Linking directory 'mydir'" message always, but print the progress bar when it is a large enough directory. E.g. we have LARGE_DIR_SIZE defined right now(https://github.com/iterative/dvc/blob/master/dvc/utils.py#L19) for deciding if we should show progress bar when computing md5 of some directory. I think that number is somewhat reasonable :slightly_smiling_face: 

Yes, progress bar should not appear in `-q` mode. We have a bug with that right now :( 

Resolving patch:
diff --git a/dvc/remote/local.py b/dvc/remote/local.py
--- a/dvc/remote/local.py
+++ b/dvc/remote/local.py
@@ -132,9 +132,13 @@ def link(self, cache, path):
         # NOTE: just create an empty file for an empty cache
         if os.path.getsize(cache) == 0:
             open(path, 'w+').close()
-            msg = "Created empty file: {} -> {}"
-            self.project.logger.info(msg.format(os.path.relpath(cache),
-                                                os.path.relpath(path)))
+
+            msg = "Created empty file: {} -> {}".format(
+                os.path.relpath(cache),
+                os.path.relpath(path),
+            )
+
+            Logger.debug(msg)
             return
 
         i = len(self.cache_types)
@@ -145,11 +149,14 @@ def link(self, cache, path):
                 if self.protected:
                     os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
 
-                msg = "Created {}'{}': {} -> {}"
-                Logger.info(msg.format('protected ' if self.protected else '',
-                                       self.cache_types[0],
-                                       os.path.relpath(cache),
-                                       os.path.relpath(path)))
+                msg = "Created {}'{}': {} -> {}".format(
+                    'protected ' if self.protected else '',
+                    self.cache_types[0],
+                    os.path.relpath(cache),
+                    os.path.relpath(path)
+                )
+
+                Logger.debug(msg)
                 return
             except DvcException as exc:
                 msg = 'Cache type \'{}\' is not supported: {}'
@@ -300,14 +307,28 @@ def checkout(self, path_info, checksum_info):
         if not os.path.exists(path):
             os.makedirs(path)
 
-        for entry in self.load_dir_cache(md5):
-            md5 = entry[self.PARAM_MD5]
-            c = self.get(md5)
+        dir_info = self.load_dir_cache(md5)
+        dir_relpath = os.path.relpath(path)
+        dir_size = len(dir_info)
+        bar = dir_size > LARGE_DIR_SIZE
+
+        Logger.info("Linking directory '{}'.".format(dir_relpath))
+
+        for processed, entry in enumerate(dir_info):
             relpath = entry[self.PARAM_RELPATH]
+            m = entry[self.PARAM_MD5]
             p = os.path.join(path, relpath)
+            c = self.get(m)
             self.link(c, p)
+
+            if bar:
+                progress.update_target(dir_relpath, processed, dir_size)
+
         self.state.update_link(path)
 
+        if bar:
+            progress.finish_target(dir_relpath)
+
     def _move(self, inp, outp):
         # moving in two stages to make last the move atomic in
         # case inp and outp are in different filesystems
@@ -335,8 +356,13 @@ def _save_file(self, path_info):
     def _save_dir(self, path_info):
         path = path_info['path']
         md5, dir_info = self.state.update_info(path)
+        dir_relpath = os.path.relpath(path)
+        dir_size = len(dir_info)
+        bar = dir_size > LARGE_DIR_SIZE
+
+        Logger.info("Linking directory '{}'.".format(dir_relpath))
 
-        for entry in dir_info:
+        for processed, entry in enumerate(dir_info):
             relpath = entry[self.PARAM_RELPATH]
             m = entry[self.PARAM_MD5]
             p = os.path.join(path, relpath)
@@ -349,8 +375,14 @@ def _save_dir(self, path_info):
 
             self.link(c, p)
 
+            if bar:
+                progress.update_target(dir_relpath, processed, dir_size)
+
         self.state.update_link(path)
 
+        if bar:
+            progress.finish_target(dir_relpath)
+
         return {self.PARAM_MD5: md5}
 
     def save(self, path_info):


Test patch: