Repository: iterative/dvc
Issue: 'Errno 24 - Too many open files' on dvc push

### Version information
* DVC version: 0.58.1
* Platform: MacOS 10.14.6
* Method of installation: pip within a conda environment

### Description
When pushing to S3 a directory of ~100 files that have been added to DVC, I observe an Errno 24 error from the dvc process.

It looks like dvc is trying to open more files than the OS allows. Checking the file handles on for the dvc process I get:

```
$ lsof -p $DVC_PID | wc -l
412
```

Looking at the OS limits, a process is limited to having 256 open files.

```
$ ulimit -a
core file size          (blocks, -c) 0
data seg size           (kbytes, -d) unlimited
file size               (blocks, -f) unlimited
max locked memory       (kbytes, -l) unlimited
max memory size         (kbytes, -m) unlimited
open files                      (-n) 256
pipe size            (512 bytes, -p) 1
stack size              (kbytes, -s) 8192
cpu time               (seconds, -t) unlimited
max user processes              (-u) 4256
virtual memory          (kbytes, -v) unlimited
```

A workaround for this is to increase the max files per process to a larger number (say 4096) by running something like `ulimit -n 4096`, but I wonder if the ideal solution is for DVC to work within the OS configured limits by default?

Edit: Updated wording of workaround

Changed files:
dvc/remote/base.py, dvc/main.py, tests/func/test_remote.py

diff --git a/dvc/main.py b/dvc/main.py
index 86e1a9e0d..89c9dc775 100644
--- a/dvc/main.py
+++ b/dvc/main.py
@@ -1,6 +1,7 @@
 """Main entry point for dvc CLI."""
 from __future__ import unicode_literals
 
+import errno
 import logging
 
 from dvc import analytics
@@ -64,6 +65,10 @@ def main(argv=None):
                 "unicode is not supported in DVC for Python 2 "
                 "(end-of-life January 1, 2020), please upgrade to Python 3"
             )
+        elif isinstance(exc, OSError) and exc.errno == errno.EMFILE:
+            logger.exception(
+                "too many open files, please increase your `ulimit`"
+            )
         else:
             logger.exception("unexpected error")
         ret = 255
diff --git a/dvc/remote/base.py b/dvc/remote/base.py
index f72e56127..57a1846a6 100644
--- a/dvc/remote/base.py
+++ b/dvc/remote/base.py
@@ -1,4 +1,7 @@
 from __future__ import unicode_literals
+
+import errno
+
 from dvc.utils.compat import basestring, FileNotFoundError, str, urlparse
 
 import itertools
@@ -516,6 +519,16 @@ class RemoteBASE(object):
             return
         self._save_file(path_info, checksum)
 
+    def _handle_transfer_exception(
+        self, from_info, to_info, exception, operation
+    ):
+        if isinstance(exception, OSError) and exception.errno == errno.EMFILE:
+            raise exception
+
+        msg = "failed to {} '{}' to '{}'".format(operation, from_info, to_info)
+        logger.exception(msg)
+        return 1
+
     def upload(self, from_info, to_info, name=None, no_progress_bar=False):
         if not hasattr(self, "_upload"):
             raise RemoteActionNotImplemented("upload", self.scheme)
@@ -537,10 +550,10 @@ class RemoteBASE(object):
                 name=name,
                 no_progress_bar=no_progress_bar,
             )
-        except Exception:
-            msg = "failed to upload '{}' to '{}'"
-            logger.exception(msg.format(from_info, to_info))
-            return 1  # 1 fail
+        except Exception as e:
+            return self._handle_transfer_exception(
+                from_info, to_info, e, "upload"
+            )
 
         return 0
 
@@ -614,10 +627,10 @@ class RemoteBASE(object):
             self._download(
                 from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
             )
-        except Exception:
-            msg = "failed to download '{}' to '{}'"
-            logger.exception(msg.format(from_info, to_info))
-            return 1  # 1 fail
+        except Exception as e:
+            return self._handle_transfer_exception(
+                from_info, to_info, e, "download"
+            )
 
         move(tmp_file, to_info, mode=file_mode)
 
diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py
index 23f318861..f3140b5f3 100644
--- a/tests/func/test_remote.py
+++ b/tests/func/test_remote.py
@@ -1,14 +1,17 @@
+import errno
 import os
 import shutil
 
 import configobj
+import pytest
 from mock import patch
 
 from dvc.config import Config
 from dvc.main import main
 from dvc.path_info import PathInfo
-from dvc.remote import RemoteLOCAL
+from dvc.remote import RemoteLOCAL, RemoteConfig
 from dvc.remote.base import RemoteBASE
+from dvc.utils.compat import fspath
 from tests.basic_env import TestDvc
 from tests.remotes import get_local_url, get_local_storagepath
 
@@ -253,3 +256,21 @@ def test_partial_push_n_pull(dvc_repo, repo_dir, caplog):
 def get_last_exc(caplog):
     _, exc, _ = caplog.records[-2].exc_info
     return exc
+
+
+def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker):
+    storage = tmp_path_factory.mktemp("test_remote_base")
+    remote_config = RemoteConfig(dvc.config)
+    remote_config.add("local_remote", fspath(storage), default=True)
+
+    tmp_dir.dvc_gen({"file": "file content"})
+
+    mocker.patch.object(
+        RemoteLOCAL,
+        "_upload",
+        side_effect=OSError(errno.EMFILE, "Too many open files"),
+    )
+
+    with pytest.raises(OSError) as e:
+        dvc.push()
+        assert e.errno == errno.EMFILE