Repository: conan-io/conan

Issue:
Cannot upload packages in which file permission has changed
Due to some workflows, developers often end up packaging up binaries that do not have write permission for `all`. When trying to fix these packages:

1. `conan download [SomeBadlyUploadedLibrary_REF]`
2. `chmod -R a+w $CONAN_USER_HOME/.conan/data/[SomeBadlyUploadedLibrary]/[Version]/[Uploader]/[Channel]/packages/*`
3. `conan upload --force -r our_remote [SomeBadlyUploadedLibrary_REF]`

*Expected*: the packages are replaced in the conan server's repository.
*Actual*: The upload step will check the hash of the package and decide that it matches so no uploading will take place.

The impact of it, is that binaries, like shared libraries without write permission cannot be imported repeatedly into our build directory, which is, while can be corrected in the local cache, it is really annoying.

Another way around is that to delete the package entirely from the remote repository, and upload it again and pray that no one is trying to pull that package right at that moment.

This could be solved a number of ways:
1. Use the file permissions in the file hashes. Problematic might be between OSes.
2. (and this might be another issue all together): the import step should first clear out the previous import if there is one, and only then do the import really. This can also help to keep the build directory cleaner, in case requirements change.
3. The `--force` argument on upload should really *force* the upload, even if everything matching. This is a user experience issue: I expect this flag to work like it works in most commands and be absolutely forceful in whatever it is doing.

I am happy to implement any or all of these, but I would like to have your opinion first.


Discussion:

Resolving patch:
diff --git a/conans/client/cmd/uploader.py b/conans/client/cmd/uploader.py
--- a/conans/client/cmd/uploader.py
+++ b/conans/client/cmd/uploader.py
@@ -373,7 +373,7 @@ def _package_files_to_upload(self, pref, policy, the_files, remote):
         self._remote_manager.check_credentials(remote)
         remote_snapshot = self._remote_manager.get_package_snapshot(pref, remote)
 
-        if remote_snapshot:
+        if remote_snapshot and policy != UPLOAD_POLICY_FORCE:
             if not is_package_snapshot_complete(remote_snapshot):
                 return the_files, set([])
             remote_manifest, _ = self._remote_manager.get_package_manifest(pref, remote)


Test patch:
diff --git a/conans/test/functional/command/upload_test.py b/conans/test/functional/command/upload_test.py
--- a/conans/test/functional/command/upload_test.py
+++ b/conans/test/functional/command/upload_test.py
@@ -1,22 +1,23 @@
-import itertools
 import os
+import platform
+import stat
 import unittest
 from collections import OrderedDict
 
+import itertools
 from mock import mock, patch
+from nose.plugins.attrib import attr
 
-
+from conans.client.cmd.uploader import CmdUpload
 from conans.client.tools.env import environment_append
 from conans.errors import ConanException
 from conans.model.ref import ConanFileReference, PackageReference
 from conans.paths import EXPORT_SOURCES_TGZ_NAME, PACKAGE_TGZ_NAME
+from conans.test.utils.conanfile import TestConanFile
 from conans.test.utils.cpp_test_files import cpp_hello_conan_files
-from conans.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, TestServer,\
-    TurboTestClient
+from conans.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, TestServer, \
+    TurboTestClient, GenConanfile
 from conans.util.files import gzopen_without_timestamps, is_dirty, save
-from conans.test.utils.conanfile import TestConanFile
-from conans.client.cmd.uploader import CmdUpload
-
 
 conanfile = """from conans import ConanFile
 class MyPkg(ConanFile):
@@ -42,6 +43,39 @@ def package(self):
 
 class UploadTest(unittest.TestCase):
 
+    @attr("artifactory_ready")
+    def test_upload_force(self):
+        ref = ConanFileReference.loads("Hello/0.1@conan/testing")
+        client = TurboTestClient(servers={"default": TestServer()})
+        pref = client.create(ref, conanfile=GenConanfile().with_package_file("myfile.sh", "foo"))
+        client.run("upload * --all --confirm")
+        self.assertIn("Uploading conan_package.tgz", client.out)
+        client.run("upload * --all --confirm")
+        self.assertNotIn("Uploading conan_package.tgz", client.out)
+
+        package_folder = client.cache.package_layout(pref.ref).package(pref)
+        package_file_path = os.path.join(package_folder, "myfile.sh")
+
+        if platform.system() == "Linux":
+            client.run("remove '*' -f")
+            client.create(ref, conanfile=GenConanfile().with_package_file("myfile.sh", "foo"))
+            os.system('chmod +x "{}"'.format(package_file_path))
+            self.assertTrue(os.stat(package_file_path).st_mode & stat.S_IXUSR)
+            client.run("upload * --all --confirm")
+            self.assertNotIn("Uploading conan_package.tgz", client.out)
+            self.assertIn("Package is up to date, upload skipped", client.out)
+            self.assertIn("Compressing package...", client.out)
+
+        client.run("upload * --all --confirm --force")
+        self.assertIn("Uploading conanfile.py", client.out)
+        self.assertIn("Uploading conan_package.tgz", client.out)
+
+        if platform.system() == "Linux":
+            client.run("remove '*' -f")
+            client.run("install {}".format(ref))
+            # Owner with execute permissions
+            self.assertTrue(os.stat(package_file_path).st_mode & stat.S_IXUSR)
+
     def test_upload_not_existing(self):
         client = TestClient(servers={"default": TestServer()},
                             users={"default": [("lasote", "mypass")]})
diff --git a/conans/test/integration/revisions_test.py b/conans/test/integration/revisions_test.py
--- a/conans/test/integration/revisions_test.py
+++ b/conans/test/integration/revisions_test.py
@@ -14,6 +14,7 @@
 from conans.util.env_reader import get_env
 
 
+@attr("artifactory_ready")
 @unittest.skipUnless(get_env("TESTING_REVISIONS_ENABLED", False), "Only revisions")
 class InstallingPackagesWithRevisionsTest(unittest.TestCase):
 
diff --git a/conans/test/utils/tools.py b/conans/test/utils/tools.py
--- a/conans/test/utils/tools.py
+++ b/conans/test/utils/tools.py
@@ -296,15 +296,13 @@ def package_exists(self, pref):
 
 class ArtifactoryServer(object):
 
-    def __init__(self, url=None, user=None, password=None, server_capabilities=None):
-        self._user = user or ARTIFACTORY_DEFAULT_USER
-        self._password = password or ARTIFACTORY_DEFAULT_PASSWORD
-        self._url = url or ARTIFACTORY_DEFAULT_URL
+    def __init__(self, *args, **kwargs):
+        self._user = ARTIFACTORY_DEFAULT_USER
+        self._password = ARTIFACTORY_DEFAULT_PASSWORD
+        self._url = ARTIFACTORY_DEFAULT_URL
         self._repo_name = "conan_{}".format(str(uuid.uuid4()).replace("-", ""))
         self.create_repository()
         self.server_store = ArtifactoryServerStore(self.repo_url, self._user, self._password)
-        if server_capabilities is not None:
-            raise nose.SkipTest("The Artifactory Server can't adjust capabilities")
 
     @property
     def _auth(self):