Repository: conan-io/conan

Issue:
[bug] Upload fails with multiple remotes in the same command
When uploading several packages at once (without specifying a remote) and some packages are uploaded to different remotes, the command will fail.

The bug appears to be due to reusing a closed thread pool in the loop in CmdUploader.upload() in uploader.py. Once thread pools are closed they can't be reused, so any iterations after the first will cause an error.

### Environment Details (include every applicable attribute)
  * Operating System+version: RHEL 7.6
  * Conan version: Tested with 1.29.0 and 1.30.dev0 (develop)
  * Python version: 3.8.5

### Steps to reproduce (Include if Applicable)
1. Set up two remotes
2. Create 2 recipes and add_ref one recipe to each remote
4. Upload using `conan upload "*" --all`

### Logs (Executed commands with output) (Include/Attach if Applicable)
I can't get the logs off my machine, but the stack trace looks like this:
command.py:2103
command.py:1477
conan_api.py:94
conan_api.py:965
uploader.py:109
pool.py:364
pool.py:473
pool.py:350
ValueError: Pool not running



Discussion:

Resolving patch:
diff --git a/conans/client/cmd/uploader.py b/conans/client/cmd/uploader.py
--- a/conans/client/cmd/uploader.py
+++ b/conans/client/cmd/uploader.py
@@ -21,6 +21,7 @@
                                gzopen_without_timestamps, set_dirty_context_manager)
 from conans.util.log import logger
 from conans.util.tracer import log_recipe_upload, log_compressed_files, log_package_upload
+from conans.tools import cpu_count
 
 
 UPLOAD_POLICY_FORCE = "force-upload"
@@ -201,12 +202,12 @@ def upload(self, reference_or_pattern, remotes, upload_recorder, package_id=None
                                            all_packages, query)
 
         if parallel_upload:
-            self._upload_thread_pool = ThreadPool(8)
             self._user_io.disable_input()
-        else:
-            self._upload_thread_pool = ThreadPool(1)
+        self._upload_thread_pool = ThreadPool(
+            cpu_count() if parallel_upload else 1)
 
         for remote, refs in refs_by_remote.items():
+
             self._output.info("Uploading to remote '{}':".format(remote.name))
 
             def upload_ref(ref_conanfile_prefs):
@@ -221,17 +222,18 @@ def upload_ref(ref_conanfile_prefs):
             self._upload_thread_pool.map(upload_ref,
                                          [(ref, conanfile, prefs) for (ref, conanfile, prefs) in
                                           refs])
-            self._upload_thread_pool.close()
-            self._upload_thread_pool.join()
-
-            if len(self._exceptions_list) > 0:
-                for exc, ref, trace in self._exceptions_list:
-                    t = "recipe" if isinstance(ref, ConanFileReference) else "package"
-                    msg = "%s: Upload %s to '%s' failed: %s\n" % (str(ref), t, remote.name, str(exc))
-                    if get_env("CONAN_VERBOSE_TRACEBACK", False):
-                        msg += trace
-                    self._output.error(msg)
-                raise ConanException("Errors uploading some packages")
+
+        self._upload_thread_pool.close()
+        self._upload_thread_pool.join()
+
+        if len(self._exceptions_list) > 0:
+            for exc, ref, trace in self._exceptions_list:
+                t = "recipe" if isinstance(ref, ConanFileReference) else "package"
+                msg = "%s: Upload %s to '%s' failed: %s\n" % (str(ref), t, remote.name, str(exc))
+                if get_env("CONAN_VERBOSE_TRACEBACK", False):
+                    msg += trace
+                self._output.error(msg)
+            raise ConanException("Errors uploading some packages")
 
         logger.debug("UPLOAD: Time manager upload: %f" % (time.time() - t1))
 


Test patch:
diff --git a/conans/test/functional/remote/multi_remote_checks_test.py b/conans/test/functional/remote/multi_remote_checks_test.py
--- a/conans/test/functional/remote/multi_remote_checks_test.py
+++ b/conans/test/functional/remote/multi_remote_checks_test.py
@@ -79,6 +79,22 @@ def package_info(self):
         client.run("remote list_ref")
         self.assertIn("Pkg/0.1@lasote/testing: server3", client.out)
 
+    def test_multiple_remotes_single_upload(self):
+        servers = OrderedDict([("server1", TestServer()),
+                               ("server2", TestServer())])
+        client = TestClient(servers=servers, users={"server1": [("lasote", "mypass")],
+                                                    "server2": [("lasote", "mypass")]})
+        conanfile = """from conans import ConanFile
+class Pkg(ConanFile):
+    settings = "build_type"
+    """
+        client.save({"conanfile.py": conanfile})
+        client.run("create . Pkg/0.1@lasote/testing -s build_type=Release")
+        client.run("create . Pkg2/0.1@lasote/testing -s build_type=Release")
+        client.run("remote add_ref Pkg/0.1@lasote/testing server1")
+        client.run("remote add_ref Pkg2/0.1@lasote/testing server2")
+        client.run("upload Pkg* --all --confirm")
+
     def test_binary_packages_mixed(self):
         servers = OrderedDict([("server1", TestServer()),
                                ("server2", TestServer()),