Repository: conan-io/conan

Issue:
An interrupted upload will not re-create an incomplete tarball
I'm using conan version 1.2.3 on Windows 10.

A few times I've had to interrupt a "conan upload" operation while it was in a "Compressing package..." step. When interrupted with Ctrl+C from the terminal, the operation leaves behind a partly-assembled tarball (i.e. the tarball does not yet have all of the package's files). The next time a "conan upload" on that same package is attempted, it appears to attempt to re-use the incomplete tarball and when I look at the package contents on the associated (Artifactory) repository I see that it's missing many files from the built package (on the build host's local cache).

An interrupted upload command should not leave an incomplete compressed source/package tarball behind.


Discussion:
Have you checked the new integrity checks released in 1.3? It seems they
address exactly your case.

Please have a look and tell me. Thanks!

El vie., 4 may. 2018 18:21, Brian Sipos <notifications@github.com> escribió:

> I'm using conan version 1.2.3 on Windows 10.
>
> A few times I've had to interrupt a "conan upload" operation while it was
> in a "Compressing package..." step. When interrupted with Ctrl+C from the
> terminal, the operation leaves behind a partly-assembled tarball (i.e. the
> tarball does not yet have all of the package's files). The next time a
> "conan upload" on that same package is attempted, it appears to attempt to
> re-use the incomplete tarball and when I look at the package contents on
> the associated (Artifactory) repository I see that it's missing many files
> from the built package (on the build host's local cache).
>
> An interrupted upload command should not leave an incomplete compressed
> source/package tarball behind.
>
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/conan-io/conan/issues/2854>, or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AOp78ICOG7l7_E-om-rCgaQ2zx9UjJ7nks5tvIAJgaJpZM4Ty65a>
> .
>

Checking this again, it seems the integrity check doesn't cover interruption of uploads.
This could be fixed capturing Ctrl+C exceptions in ``compress_files()``, but that would still leave out the case when the process is completely killed (-9). 

There exists the ``conan upload --check`` argument, which probably could be extended to check for the tgz integrity. 
Let's try to improve it for the next release.

Resolving patch:
diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py
--- a/conans/client/remote_manager.py
+++ b/conans/client/remote_manager.py
@@ -12,7 +12,7 @@
 from conans.paths import PACKAGE_TGZ_NAME, CONANINFO, CONAN_MANIFEST, CONANFILE, EXPORT_TGZ_NAME, \
     rm_conandir, EXPORT_SOURCES_TGZ_NAME, EXPORT_SOURCES_DIR_OLD
 from conans.util.files import gzopen_without_timestamps, is_dirty,\
-    make_read_only
+    make_read_only, set_dirty, clean_dirty
 from conans.util.files import tar_extract, rmdir, exception_message_safe, mkdir
 from conans.util.files import touch_folder
 from conans.util.log import logger
@@ -39,6 +39,14 @@ def upload_recipe(self, conan_reference, remote, retry, retry_wait, ignore_delet
 
         t1 = time.time()
         export_folder = self._client_cache.export(conan_reference)
+
+        for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME):
+            tgz_path = os.path.join(export_folder, f)
+            if is_dirty(tgz_path):
+                self._output.warn("%s: Removing %s, marked as dirty" % (str(conan_reference), f))
+                os.remove(tgz_path)
+                clean_dirty(tgz_path)
+
         files, symlinks = gather_files(export_folder)
         if CONANFILE not in files or CONAN_MANIFEST not in files:
             raise ConanException("Cannot upload corrupted recipe '%s'" % str(conan_reference))
@@ -99,6 +107,11 @@ def upload_package(self, package_reference, remote, retry, retry_wait, skip_uplo
                                  "Remove it with 'conan remove %s -p=%s'" % (package_reference,
                                                                              package_reference.conan,
                                                                              package_reference.package_id))
+        tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
+        if is_dirty(tgz_path):
+            self._output.warn("%s: Removing %s, marked as dirty" % (str(package_reference), PACKAGE_TGZ_NAME))
+            os.remove(tgz_path)
+            clean_dirty(tgz_path)
         # Get all the files in that directory
         files, symlinks = gather_files(package_folder)
 
@@ -329,6 +342,7 @@ def compress_files(files, symlinks, name, dest_dir):
     t1 = time.time()
     # FIXME, better write to disk sequentially and not keep tgz contents in memory
     tgz_path = os.path.join(dest_dir, name)
+    set_dirty(tgz_path)
     with open(tgz_path, "wb") as tgz_handle:
         # tgz_contents = BytesIO()
         tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle)
@@ -354,6 +368,7 @@ def compress_files(files, symlinks, name, dest_dir):
 
         tgz.close()
 
+    clean_dirty(tgz_path)
     duration = time.time() - t1
     log_compressed_files(files, duration, tgz_path)
 


Test patch:
diff --git a/conans/test/command/upload_test.py b/conans/test/command/upload_test.py
--- a/conans/test/command/upload_test.py
+++ b/conans/test/command/upload_test.py
@@ -2,9 +2,12 @@
 from conans.tools import environment_append
 from conans.test.utils.tools import TestClient, TestServer
 from conans.test.utils.cpp_test_files import cpp_hello_conan_files
-from conans.model.ref import ConanFileReference
-from conans.util.files import save
+from conans.model.ref import ConanFileReference, PackageReference
+from conans.util.files import save, is_dirty, gzopen_without_timestamps
 import os
+from mock import mock
+from conans.errors import ConanException
+from conans.paths import EXPORT_SOURCES_TGZ_NAME, PACKAGE_TGZ_NAME
 
 
 conanfile = """from conans import ConanFile
@@ -75,6 +78,62 @@ def pattern_upload_test(self):
         self.assertIn("Uploading conan_package.tgz", client.user_io.out)
         self.assertIn("Uploading conanfile.py", client.user_io.out)
 
+    def broken_sources_tgz_test(self):
+        # https://github.com/conan-io/conan/issues/2854
+        client = self._client()
+        client.save({"conanfile.py": conanfile,
+                     "source.h": "my source"})
+        client.run("create . user/testing")
+        ref = ConanFileReference.loads("Hello0/1.2.1@user/testing")
+
+        def gzopen_patched(name, mode="r", fileobj=None, compresslevel=None, **kwargs):
+            raise ConanException("Error gzopen %s" % name)
+        with mock.patch('conans.client.remote_manager.gzopen_without_timestamps', new=gzopen_patched):
+            error = client.run("upload * --confirm", ignore_error=True)
+            self.assertTrue(error)
+            self.assertIn("ERROR: Error gzopen conan_sources.tgz", client.out)
+
+            export_folder = client.client_cache.export(ref)
+            tgz = os.path.join(export_folder, EXPORT_SOURCES_TGZ_NAME)
+            self.assertTrue(os.path.exists(tgz))
+            self.assertTrue(is_dirty(tgz))
+
+        client.run("upload * --confirm")
+        self.assertIn("WARN: Hello0/1.2.1@user/testing: Removing conan_sources.tgz, marked as dirty",
+                      client.out)
+        self.assertTrue(os.path.exists(tgz))
+        self.assertFalse(is_dirty(tgz))
+
+    def broken_package_tgz_test(self):
+        # https://github.com/conan-io/conan/issues/2854
+        client = self._client()
+        client.save({"conanfile.py": conanfile,
+                     "source.h": "my source"})
+        client.run("create . user/testing")
+        package_ref = PackageReference.loads("Hello0/1.2.1@user/testing:"
+                                             "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9")
+
+        def gzopen_patched(name, mode="r", fileobj=None, compresslevel=None, **kwargs):
+            if name == PACKAGE_TGZ_NAME:
+                raise ConanException("Error gzopen %s" % name)
+            return gzopen_without_timestamps(name, mode, fileobj, compresslevel, **kwargs)
+        with mock.patch('conans.client.remote_manager.gzopen_without_timestamps', new=gzopen_patched):
+            error = client.run("upload * --confirm --all", ignore_error=True)
+            self.assertTrue(error)
+            self.assertIn("ERROR: Error gzopen conan_package.tgz", client.out)
+
+            export_folder = client.client_cache.package(package_ref)
+            tgz = os.path.join(export_folder, PACKAGE_TGZ_NAME)
+            self.assertTrue(os.path.exists(tgz))
+            self.assertTrue(is_dirty(tgz))
+
+        client.run("upload * --confirm --all")
+        self.assertIn("WARN: Hello0/1.2.1@user/testing:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9: "
+                      "Removing conan_package.tgz, marked as dirty",
+                      client.out)
+        self.assertTrue(os.path.exists(tgz))
+        self.assertFalse(is_dirty(tgz))
+
     def corrupt_upload_test(self):
         client = self._client()