Repository: conan-io/conan

Issue:
Store checksum of downloaded (and uploaded) packages
Store in the ``metadata.json`` the checksum of the files actually downloaded or uploaded to the servers. We need this data to generate the _build information_ without querying the server again. For Artifactory, the required properties would be ``actual_md5`` and ``actual_sha1``.




Discussion:

Resolving patch:
diff --git a/conans/client/cmd/uploader.py b/conans/client/cmd/uploader.py
--- a/conans/client/cmd/uploader.py
+++ b/conans/client/cmd/uploader.py
@@ -5,7 +5,7 @@
 from collections import defaultdict
 
 from conans.util import progress_bar
-from conans.client.remote_manager import is_package_snapshot_complete
+from conans.client.remote_manager import is_package_snapshot_complete, calc_files_checksum
 from conans.client.source import complete_recipe_sources
 from conans.errors import ConanException, NotFoundException
 from conans.model.manifest import gather_files, FileTreeManifest
@@ -219,6 +219,10 @@ def _upload_recipe(self, ref, conanfile, retry, retry_wait, policy, remote, remo
 
         t1 = time.time()
         the_files = self._compress_recipe_files(ref)
+
+        with self._cache.package_layout(ref).update_metadata() as metadata:
+            metadata.recipe.checksums = calc_files_checksum(the_files)
+
         local_manifest = FileTreeManifest.loads(load(the_files["conanmanifest.txt"]))
 
         remote_manifest = None
@@ -263,6 +267,10 @@ def _upload_package(self, pref, retry=None, retry_wait=None, integrity_check=Fal
 
         t1 = time.time()
         the_files = self._compress_package_files(pref, integrity_check)
+
+        with self._cache.package_layout(pref.ref).update_metadata() as metadata:
+            metadata.packages[pref.id].checksums = calc_files_checksum(the_files)
+
         if policy == UPLOAD_POLICY_SKIP:
             return None
         files_to_upload, deleted = self._package_files_to_upload(pref, policy, the_files, p_remote)
@@ -306,6 +314,7 @@ def _compress_recipe_files(self, ref):
         src_files, src_symlinks = gather_files(export_src_folder)
         the_files = _compress_recipe_files(files, symlinks, src_files, src_symlinks, export_folder,
                                            self._output)
+
         return the_files
 
     def _compress_package_files(self, pref, integrity_check):
diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py
--- a/conans/client/remote_manager.py
+++ b/conans/client/remote_manager.py
@@ -15,7 +15,7 @@
 from conans.util import progress_bar
 from conans.util.env_reader import get_env
 from conans.util.files import make_read_only, mkdir, rmdir, tar_extract, touch_folder, \
-    merge_directories
+    merge_directories, md5sum, sha1sum
 from conans.util.log import logger
 # FIXME: Eventually, when all output is done, tracer functions should be moved to the recorder class
 from conans.util.tracer import (log_package_download,
@@ -87,6 +87,8 @@ def get_recipe(self, ref, remote):
         duration = time.time() - t1
         log_recipe_download(ref, duration, remote.name, zipped_files)
 
+        recipe_checksums = calc_files_checksum(zipped_files)
+
         unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME, output=self._output)
         # Make sure that the source dir is deleted
         package_layout = self._cache.package_layout(ref)
@@ -96,6 +98,7 @@ def get_recipe(self, ref, remote):
 
         with package_layout.update_metadata() as metadata:
             metadata.recipe.revision = ref.revision
+            metadata.recipe.checksums = recipe_checksums
 
         self._hook_manager.execute("post_download_recipe", conanfile_path=conanfile_path,
                                    reference=ref, remote=remote)
@@ -138,9 +141,12 @@ def get_package(self, pref, dest_folder, remote, output, recorder):
                 raise PackageNotFoundException(pref)
             zipped_files = self._call_remote(remote, "get_package", pref, dest_folder)
 
+            package_checksums = calc_files_checksum(zipped_files)
+
             with self._cache.package_layout(pref.ref).update_metadata() as metadata:
                 metadata.packages[pref.id].revision = pref.revision
                 metadata.packages[pref.id].recipe_revision = pref.ref.revision
+                metadata.packages[pref.id].checksums = package_checksums
 
             duration = time.time() - t1
             log_package_download(pref, duration, remote, zipped_files)
@@ -248,6 +254,10 @@ def _call_remote(self, remote, method, *argc, **argv):
             raise ConanException(exc, remote=remote)
 
 
+def calc_files_checksum(files):
+    return {file_name: {"md5": md5sum(path), "sha1": sha1sum(path)} for file_name, path in files.items()}
+
+
 def is_package_snapshot_complete(snapshot):
     integrity = True
     for keyword in ["conaninfo", "conanmanifest", "conan_package"]:
diff --git a/conans/model/package_metadata.py b/conans/model/package_metadata.py
--- a/conans/model/package_metadata.py
+++ b/conans/model/package_metadata.py
@@ -9,6 +9,7 @@ class _RecipeMetadata(object):
     def __init__(self):
         self._revision = None
         self.properties = {}
+        self.checksums = {}
         self.remote = None
 
     @property
@@ -22,7 +23,8 @@ def revision(self, r):
     def to_dict(self):
         ret = {"revision": self.revision,
                "remote": self.remote,
-               "properties": self.properties}
+               "properties": self.properties,
+               "checksums": self.checksums}
         return ret
 
     @staticmethod
@@ -31,6 +33,7 @@ def loads(data):
         ret.revision = data["revision"]
         ret.remote = data.get("remote")
         ret.properties = data["properties"]
+        ret.checksums = data.get("checksums", {})
         ret.time = data.get("time")
         return ret
 
@@ -41,6 +44,7 @@ def __init__(self):
         self._revision = None
         self._recipe_revision = None
         self.properties = {}
+        self.checksums = {}
         self.remote = None
 
     @property
@@ -63,7 +67,8 @@ def to_dict(self):
         ret = {"revision": self.revision,
                "recipe_revision": self.recipe_revision,
                "remote": self.remote,
-               "properties": self.properties}
+               "properties": self.properties,
+               "checksums": self.checksums}
         return ret
 
     @staticmethod
@@ -72,6 +77,7 @@ def loads(data):
         ret.revision = data.get("revision")
         ret.recipe_revision = data.get("recipe_revision")
         ret.properties = data.get("properties")
+        ret.checksums = data.get("checksums", {})
         ret.remote = data.get("remote")
         return ret
 


Test patch:
diff --git a/conans/test/functional/command/upload_test.py b/conans/test/functional/command/upload_test.py
--- a/conans/test/functional/command/upload_test.py
+++ b/conans/test/functional/command/upload_test.py
@@ -749,6 +749,34 @@ def upload_without_user_channel_test(self):
         self.assertIn("Uploading package 1/1: 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 to 'default'",
                       client.out)
 
+    def checksums_metadata_test(self):
+        client = TestClient(default_server_user=True)
+        client.save({"conanfile.py": GenConanfile()})
+        client.run('create . lib/1.0@user/channel')
+        client.run('upload lib/1.0 -c --all -r default')
+        ref = ConanFileReference("lib", "1.0", "user", "channel")
+        metadata = client.cache.package_layout(ref).load_metadata()
+        package_md5 = metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"]["md5"]
+        package_sha1 = metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"][
+            "sha1"]
+        recipe_md5 = metadata.recipe.checksums["conanfile.py"]["md5"]
+        recipe_sha1 = metadata.recipe.checksums["conanfile.py"]["sha1"]
+        self.assertEqual(package_md5, "25f53ac9685e07815b990e7c6f21bfd0")
+        self.assertEqual(package_sha1, "be152c82859285658a06816aaaec529a159c33d3")
+        self.assertEqual(recipe_md5, "8eda41e0997f3eacc436fb6c621d7396")
+        self.assertEqual(recipe_sha1, "b97d6b26be5bd02252a44c265755f873cf5ec70b")
+        client.run('remove * -f')
+        client.run('install lib/1.0@user/channel -r default')
+        metadata = client.cache.package_layout(ref).load_metadata()
+        self.assertEqual(
+            metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"]["md5"],
+            package_md5)
+        self.assertEqual(
+            metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"]["sha1"],
+            package_sha1)
+        self.assertEqual(metadata.recipe.checksums["conanfile.py"]["md5"], recipe_md5)
+        self.assertEqual(metadata.recipe.checksums["conanfile.py"]["sha1"], recipe_sha1)
+        
     def upload_without_cleaned_user_test(self):
         """ When a user is not authenticated, uploads failed first time
         https://github.com/conan-io/conan/issues/5878
diff --git a/conans/test/unittests/model/package_metadata_test.py b/conans/test/unittests/model/package_metadata_test.py
--- a/conans/test/unittests/model/package_metadata_test.py
+++ b/conans/test/unittests/model/package_metadata_test.py
@@ -8,9 +8,13 @@ class PackageMetadataTest(unittest.TestCase):
     def test_load_unload(self):
         a = PackageMetadata()
         a.recipe.revision = "rev"
+        a.recipe.checksums["somefile1"] = {"md5": "50b2137a5d63567b7e88b743a3b594cf",
+                                           "sha1": "0b7e8ed59ff4eacb95fd3cc8e17a8034584a96c2"}
         a.packages["ID"].recipe_revision = "rec_rev"
         a.packages["ID"].revision = "revp"
         a.packages["ID"].properties["Someprop"] = "23"
+        a.packages["ID"].checksums["somefile2"] = {"md5": "efb7597b146344532fe8da2b79860aaa",
+                                                   "sha1": "cc3e6eae41eca26538630f4cd5b0bf4fb52e2d"}
 
         tmp = a.dumps()
 
@@ -18,6 +22,10 @@ def test_load_unload(self):
 
         self.assertEqual(b, a)
         self.assertEqual(b.packages["ID"].properties["Someprop"], "23")
+        self.assertEqual(b.recipe.checksums["somefile1"]["md5"],
+                         "50b2137a5d63567b7e88b743a3b594cf")
+        self.assertEqual(b.packages["ID"].checksums["somefile2"]["sha1"],
+                         "cc3e6eae41eca26538630f4cd5b0bf4fb52e2d")
 
     def test_other_types_than_str(self):
         a = PackageMetadata()